[llvm] [AArch64][SME]Check streaming mode when using SME2 instruction in fra… (PR #109680)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 27 07:07:37 PDT 2024
https://github.com/CarolineConcatto updated https://github.com/llvm/llvm-project/pull/109680
>From 2ce1be225eb09b89b8efceeea0338b20b072e9d8 Mon Sep 17 00:00:00 2001
From: Caroline Concatto <caroline.concatto at arm.com>
Date: Mon, 23 Sep 2024 13:37:41 +0000
Subject: [PATCH 1/4] [AArch64][SME]Check streaming mode when using SME2
instruction in frame lowering
SME instructions can only be used in streaming mode.
PTRUE for predicated counter and the ld/st pair can be used when:
sve2.1 is available or
sme2 available in function in streaming mode.
Previously the frame lowering only checking if sme2 available when building
the machine instruction.
This fix checks if sme2 is available and is subtarget in streaming mode
---
llvm/lib/Target/AArch64/AArch64FrameLowering.cpp | 14 +++++++++++---
.../AArch64/sve-callee-save-restore-pairs.ll | 3 ++-
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index ad20e76d0fe2e0..9849e35bcb266b 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -2955,6 +2955,14 @@ unsigned findFreePredicateReg(BitVector &SavedRegs) {
return AArch64::NoRegister;
}
+bool isSve2p1OrSme2InStreaming(const AArch64Subtarget &Subtarget) {
+ if (Subtarget.hasSVE2p1())
+ return true;
+ if (Subtarget.hasSME2() && Subtarget.isStreaming())
+ return true;
+ return false;
+}
+
static void computeCalleeSaveRegisterPairs(
MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI,
const TargetRegisterInfo *TRI, SmallVectorImpl<RegPairInfo> &RegPairs,
@@ -3326,7 +3334,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
MF.getSubtarget<AArch64Subtarget>();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
- assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
+ assert((PnReg != 0 && isSve2p1OrSme2InStreaming(Subtarget)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
auto IsPPR = [](const RegPairInfo &c) {
@@ -3504,7 +3512,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
- assert(((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) && PnReg != 0) &&
+ assert((PnReg != 0 && isSve2p1OrSme2InStreaming(Subtarget)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
assert(!(PPRBegin < ZPRBegin) &&
@@ -3718,7 +3726,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
SavedRegs.test(CSRegs[i ^ 1]));
}
- if (HasPairZReg && (Subtarget.hasSVE2p1() || Subtarget.hasSME2())) {
+ if (HasPairZReg && isSve2p1OrSme2InStreaming(Subtarget)) {
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
// Find a suitable predicate register for the multi-vector spill/fill
// instructions.
diff --git a/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll b/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
index 470c0dd45782c9..9b5cd3251c22a2 100644
--- a/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
+++ b/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
@@ -3,7 +3,8 @@
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=NOPAIR
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs -force-streaming < %s | FileCheck %s --check-prefixes=PAIR
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=PAIR
-
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sve2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=NOPAIR
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sve2 -verify-machineinstrs -force-streaming < %s | FileCheck %s --check-prefixes=PAIR
declare void @my_func()
>From ea18f5f7a6965ade45e59dc89d49ac95aa0cae8a Mon Sep 17 00:00:00 2001
From: Caroline Concatto <caroline.concatto at arm.com>
Date: Wed, 25 Sep 2024 17:01:45 +0000
Subject: [PATCH 2/4] Address review comments
This patch:
- Replace isSve2p1OrSme2InStreaming by enableMultiVectorSpillFill
- adds a aarch64-disable-multivector-spill-fill clang option
- add a test for streaming compatible function
---
.../Target/AArch64/AArch64FrameLowering.cpp | 29 +-
.../AArch64/sme-callee-save-restore-pairs.ll | 306 ++++++++++++++++++
.../AArch64/sve-callee-save-restore-pairs.ll | 9 +-
3 files changed, 332 insertions(+), 12 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 9849e35bcb266b..42967092216900 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -285,6 +285,11 @@ static cl::opt<bool>
StackHazardInNonStreaming("aarch64-stack-hazard-in-non-streaming",
cl::init(false), cl::Hidden);
+static cl::opt<bool> DisableMultiVectorSpillFill(
+ "aarch64-disable-multivector-spill-fill",
+ cl::desc("Disable use of LD/ST pairs for SME2 or SVE2p1"), cl::init(false),
+ cl::Hidden);
+
STATISTIC(NumRedZoneFunctions, "Number of functions using red zone");
/// Returns how much of the incoming argument stack area (in bytes) we should
@@ -2955,12 +2960,18 @@ unsigned findFreePredicateReg(BitVector &SavedRegs) {
return AArch64::NoRegister;
}
-bool isSve2p1OrSme2InStreaming(const AArch64Subtarget &Subtarget) {
- if (Subtarget.hasSVE2p1())
- return true;
- if (Subtarget.hasSME2() && Subtarget.isStreaming())
- return true;
- return false;
+// The multivector LD/ST are available only for SME or SVE2p1 targets
+bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget,
+ MachineFunction &MF) {
+ if (DisableMultiVectorSpillFill)
+ return false;
+
+ SMEAttrs Attrs(MF.getFunction());
+ bool LocallyStreaming =
+ Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface();
+
+ return Subtarget.hasSVE2p1() ||
+ (Subtarget.hasSME2() && (Subtarget.isStreaming() || LocallyStreaming));
}
static void computeCalleeSaveRegisterPairs(
@@ -3334,7 +3345,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
MF.getSubtarget<AArch64Subtarget>();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
- assert((PnReg != 0 && isSve2p1OrSme2InStreaming(Subtarget)) &&
+ assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget, MF)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
auto IsPPR = [](const RegPairInfo &c) {
@@ -3512,7 +3523,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
- assert((PnReg != 0 && isSve2p1OrSme2InStreaming(Subtarget)) &&
+ assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget, MF)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
assert(!(PPRBegin < ZPRBegin) &&
@@ -3726,7 +3737,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
SavedRegs.test(CSRegs[i ^ 1]));
}
- if (HasPairZReg && isSve2p1OrSme2InStreaming(Subtarget)) {
+ if (HasPairZReg && enableMultiVectorSpillFill(Subtarget, MF)) {
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
// Find a suitable predicate register for the multi-vector spill/fill
// instructions.
diff --git a/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll b/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
new file mode 100644
index 00000000000000..3faf1485d18a72
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
@@ -0,0 +1,306 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sve -aarch64-disable-multivector-spill-fill -verify-machineinstrs < %s | FileCheck %s --check-prefixes=NOPAIR
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -mattr=+sve -verify-machineinstrs < %s | FileCheck %s --check-prefixes=NOPAIR
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sve -verify-machineinstrs < %s | FileCheck %s --check-prefixes=PAIR
+
+declare void @my_func()
+declare void @my_func2(<vscale x 16 x i8> %v)
+
+define void @fbyte(<vscale x 16 x i8> %v) #0{
+; NOPAIR-LABEL: fbyte:
+; NOPAIR: // %bb.0:
+; NOPAIR-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; NOPAIR-NEXT: cntd x9
+; NOPAIR-NEXT: stp x9, x19, [sp, #16] // 16-byte Folded Spill
+; NOPAIR-NEXT: addvl sp, sp, #-18
+; NOPAIR-NEXT: str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: addvl sp, sp, #-1
+; NOPAIR-NEXT: str z0, [sp] // 16-byte Folded Spill
+; NOPAIR-NEXT: bl __arm_sme_state
+; NOPAIR-NEXT: and x19, x0, #0x1
+; NOPAIR-NEXT: tbz w19, #0, .LBB0_2
+; NOPAIR-NEXT: // %bb.1:
+; NOPAIR-NEXT: smstop sm
+; NOPAIR-NEXT: .LBB0_2:
+; NOPAIR-NEXT: ldr z0, [sp] // 16-byte Folded Reload
+; NOPAIR-NEXT: bl my_func2
+; NOPAIR-NEXT: tbz w19, #0, .LBB0_4
+; NOPAIR-NEXT: // %bb.3:
+; NOPAIR-NEXT: smstart sm
+; NOPAIR-NEXT: .LBB0_4:
+; NOPAIR-NEXT: addvl sp, sp, #1
+; NOPAIR-NEXT: ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: addvl sp, sp, #18
+; NOPAIR-NEXT: ldr x19, [sp, #24] // 8-byte Folded Reload
+; NOPAIR-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; NOPAIR-NEXT: ret
+;
+; PAIR-LABEL: fbyte:
+; PAIR: // %bb.0:
+; PAIR-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; PAIR-NEXT: cntd x9
+; PAIR-NEXT: stp x9, x19, [sp, #16] // 16-byte Folded Spill
+; PAIR-NEXT: addvl sp, sp, #-18
+; PAIR-NEXT: str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: addvl sp, sp, #-1
+; PAIR-NEXT: str z0, [sp] // 16-byte Folded Spill
+; PAIR-NEXT: bl __arm_sme_state
+; PAIR-NEXT: and x19, x0, #0x1
+; PAIR-NEXT: tbz w19, #0, .LBB0_2
+; PAIR-NEXT: // %bb.1:
+; PAIR-NEXT: smstop sm
+; PAIR-NEXT: .LBB0_2:
+; PAIR-NEXT: ldr z0, [sp] // 16-byte Folded Reload
+; PAIR-NEXT: bl my_func2
+; PAIR-NEXT: tbz w19, #0, .LBB0_4
+; PAIR-NEXT: // %bb.3:
+; PAIR-NEXT: smstart sm
+; PAIR-NEXT: .LBB0_4:
+; PAIR-NEXT: addvl sp, sp, #1
+; PAIR-NEXT: ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: addvl sp, sp, #18
+; PAIR-NEXT: ldr x19, [sp, #24] // 8-byte Folded Reload
+; PAIR-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; PAIR-NEXT: ret
+ call void @my_func2(<vscale x 16 x i8> %v)
+ ret void
+}
+
+define void @fhalf(<vscale x 8 x half> %v) #1{
+; NOPAIR-LABEL: fhalf:
+; NOPAIR: // %bb.0:
+; NOPAIR-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; NOPAIR-NEXT: cntd x9
+; NOPAIR-NEXT: str x9, [sp, #16] // 8-byte Folded Spill
+; NOPAIR-NEXT: addvl sp, sp, #-18
+; NOPAIR-NEXT: str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: smstop sm
+; NOPAIR-NEXT: bl my_func
+; NOPAIR-NEXT: smstart sm
+; NOPAIR-NEXT: ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: addvl sp, sp, #18
+; NOPAIR-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; NOPAIR-NEXT: ret
+;
+; PAIR-LABEL: fhalf:
+; PAIR: // %bb.0:
+; PAIR-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; PAIR-NEXT: cntd x9
+; PAIR-NEXT: str x9, [sp, #16] // 8-byte Folded Spill
+; PAIR-NEXT: addvl sp, sp, #-18
+; PAIR-NEXT: str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: ptrue pn8.b
+; PAIR-NEXT: str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: st1b { z22.b, z23.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #16, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #20, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #28, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #32, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: smstop sm
+; PAIR-NEXT: bl my_func
+; PAIR-NEXT: smstart sm
+; PAIR-NEXT: ptrue pn8.b
+; PAIR-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ld1b { z22.b, z23.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #16, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #20, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #28, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #32, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: addvl sp, sp, #18
+; PAIR-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; PAIR-NEXT: ret
+ call void @my_func()
+ ret void
+}
+
+attributes #0 = { nounwind "aarch64_pstate_sm_compatible" }
+attributes #1 = { nounwind "aarch64_pstate_sm_enabled" }
diff --git a/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll b/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
index 9b5cd3251c22a2..c3cf628f13dc9f 100644
--- a/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
+++ b/llvm/test/CodeGen/AArch64/sve-callee-save-restore-pairs.ll
@@ -1,14 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -aarch64-disable-multivector-spill-fill -verify-machineinstrs -force-streaming < %s | FileCheck %s --check-prefixes=NOPAIR
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -aarch64-disable-multivector-spill-fill -verify-machineinstrs < %s | FileCheck %s --check-prefixes=NOPAIR
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme -verify-machineinstrs -force-streaming < %s | FileCheck %s --check-prefixes=NOPAIR
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=NOPAIR
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -verify-machineinstrs -force-streaming < %s | FileCheck %s --check-prefixes=PAIR
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve2p1 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=PAIR
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sve2 -verify-machineinstrs < %s | FileCheck %s --check-prefixes=NOPAIR
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sve2 -verify-machineinstrs -force-streaming < %s | FileCheck %s --check-prefixes=PAIR
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sve -verify-machineinstrs < %s | FileCheck %s --check-prefixes=NOPAIR
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -mattr=+sve -verify-machineinstrs -force-streaming < %s | FileCheck %s --check-prefixes=PAIR
declare void @my_func()
-define void @fbyte(<vscale x 16 x i8> %v) {
+define void @fbyte(<vscale x 16 x i8> %v){
; NOPAIR-LABEL: fbyte:
; NOPAIR: // %bb.0:
; NOPAIR-NEXT: stp x29, x30, [sp, #-16]! // 16-byte Folded Spill
@@ -555,3 +557,4 @@ define aarch64_sve_vector_pcs void @test_clobbers_p_reg_negative() {
call void asm sideeffect "", "~{p10}"()
ret void
}
+
>From 0d8fe183686a7e088781d621867f3dcd15332942 Mon Sep 17 00:00:00 2001
From: Caroline Concatto <caroline.concatto at arm.com>
Date: Thu, 26 Sep 2024 13:02:19 +0000
Subject: [PATCH 3/4] Add locally streaming test for sme
---
.../Target/AArch64/AArch64FrameLowering.cpp | 15 +-
.../AArch64/sme-callee-save-restore-pairs.ll | 137 ++++++++++++++++++
2 files changed, 142 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 42967092216900..43b6bde693ab1c 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -2961,17 +2961,12 @@ unsigned findFreePredicateReg(BitVector &SavedRegs) {
}
// The multivector LD/ST are available only for SME or SVE2p1 targets
-bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget,
- MachineFunction &MF) {
+bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget) {
if (DisableMultiVectorSpillFill)
return false;
- SMEAttrs Attrs(MF.getFunction());
- bool LocallyStreaming =
- Attrs.hasStreamingBody() && !Attrs.hasStreamingInterface();
-
return Subtarget.hasSVE2p1() ||
- (Subtarget.hasSME2() && (Subtarget.isStreaming() || LocallyStreaming));
+ (Subtarget.hasSME2() && Subtarget.isStreaming());
}
static void computeCalleeSaveRegisterPairs(
@@ -3345,7 +3340,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
MF.getSubtarget<AArch64Subtarget>();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
- assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget, MF)) &&
+ assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
auto IsPPR = [](const RegPairInfo &c) {
@@ -3523,7 +3518,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
- assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget, MF)) &&
+ assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
assert(!(PPRBegin < ZPRBegin) &&
@@ -3737,7 +3732,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
SavedRegs.test(CSRegs[i ^ 1]));
}
- if (HasPairZReg && enableMultiVectorSpillFill(Subtarget, MF)) {
+ if (HasPairZReg && enableMultiVectorSpillFill(Subtarget)) {
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
// Find a suitable predicate register for the multi-vector spill/fill
// instructions.
diff --git a/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll b/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
index 3faf1485d18a72..9ba6a9db4cef4d 100644
--- a/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
+++ b/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
@@ -302,5 +302,142 @@ define void @fhalf(<vscale x 8 x half> %v) #1{
ret void
}
+define void @ffloat(<vscale x 4 x i32> %v) #2 {
+; NOPAIR-LABEL: ffloat:
+; NOPAIR: // %bb.0:
+; NOPAIR-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; NOPAIR-NEXT: rdsvl x9, #1
+; NOPAIR-NEXT: lsr x9, x9, #3
+; NOPAIR-NEXT: str x9, [sp, #16] // 8-byte Folded Spill
+; NOPAIR-NEXT: cntd x9
+; NOPAIR-NEXT: str x9, [sp, #24] // 8-byte Folded Spill
+; NOPAIR-NEXT: addsvl sp, sp, #-18
+; NOPAIR-NEXT: str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; NOPAIR-NEXT: str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; NOPAIR-NEXT: smstart sm
+; NOPAIR-NEXT: smstop sm
+; NOPAIR-NEXT: bl my_func
+; NOPAIR-NEXT: ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; NOPAIR-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; NOPAIR-NEXT: addsvl sp, sp, #18
+; NOPAIR-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; NOPAIR-NEXT: ret
+;
+; PAIR-LABEL: ffloat:
+; PAIR: // %bb.0:
+; PAIR-NEXT: stp x29, x30, [sp, #-32]! // 16-byte Folded Spill
+; PAIR-NEXT: rdsvl x9, #1
+; PAIR-NEXT: lsr x9, x9, #3
+; PAIR-NEXT: str x9, [sp, #16] // 8-byte Folded Spill
+; PAIR-NEXT: cntd x9
+; PAIR-NEXT: str x9, [sp, #24] // 8-byte Folded Spill
+; PAIR-NEXT: addsvl sp, sp, #-18
+; PAIR-NEXT: str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: ptrue pn8.b
+; PAIR-NEXT: str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: st1b { z22.b, z23.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #16, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #20, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #28, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #32, mul vl] // 32-byte Folded Spill
+; PAIR-NEXT: str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: smstart sm
+; PAIR-NEXT: smstop sm
+; PAIR-NEXT: bl my_func
+; PAIR-NEXT: ptrue pn8.b
+; PAIR-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ld1b { z22.b, z23.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #16, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #20, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #28, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #32, mul vl] // 32-byte Folded Reload
+; PAIR-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; PAIR-NEXT: addsvl sp, sp, #18
+; PAIR-NEXT: ldp x29, x30, [sp], #32 // 16-byte Folded Reload
+; PAIR-NEXT: ret
+ call void @my_func()
+ ret void
+}
+
+
+
attributes #0 = { nounwind "aarch64_pstate_sm_compatible" }
attributes #1 = { nounwind "aarch64_pstate_sm_enabled" }
+attributes #2 = { nounwind "aarch64_pstate_sm_body" }
>From a44bd501ee8f8ec689cddbc13719d4dce6f96d30 Mon Sep 17 00:00:00 2001
From: Caroline Concatto <caroline.concatto at arm.com>
Date: Fri, 27 Sep 2024 09:54:48 +0000
Subject: [PATCH 4/4] Avoid use of sme2 instructions when in locally streaming
mode
---
.../Target/AArch64/AArch64FrameLowering.cpp | 19 +++++--
.../AArch64/sme-callee-save-restore-pairs.ll | 52 ++++++++++++-------
2 files changed, 47 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 43b6bde693ab1c..aa4ce1889c7cd2 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -2961,12 +2961,21 @@ unsigned findFreePredicateReg(BitVector &SavedRegs) {
}
// The multivector LD/ST are available only for SME or SVE2p1 targets
-bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget) {
+bool enableMultiVectorSpillFill(const AArch64Subtarget &Subtarget,
+ MachineFunction &MF) {
if (DisableMultiVectorSpillFill)
return false;
+ SMEAttrs FuncAttrs(MF.getFunction());
+ bool IsLocallyStreaming =
+ FuncAttrs.hasStreamingBody() && !FuncAttrs.hasStreamingInterface();
+
+ // Only when in streaming mode SME2 instructions can be safely used.
+ // It is not safe to use SME2 instructions when in streaming compatible or
+ // locally streaming mode.
return Subtarget.hasSVE2p1() ||
- (Subtarget.hasSME2() && Subtarget.isStreaming());
+ (Subtarget.hasSME2() &&
+ (!IsLocallyStreaming && Subtarget.isStreaming()));
}
static void computeCalleeSaveRegisterPairs(
@@ -3340,7 +3349,7 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
MF.getSubtarget<AArch64Subtarget>();
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
- assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget)) &&
+ assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget, MF)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
auto IsPPR = [](const RegPairInfo &c) {
@@ -3518,7 +3527,7 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
[[maybe_unused]] const AArch64Subtarget &Subtarget =
MF.getSubtarget<AArch64Subtarget>();
unsigned PnReg = AFI->getPredicateRegForFillSpill();
- assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget)) &&
+ assert((PnReg != 0 && enableMultiVectorSpillFill(Subtarget, MF)) &&
"Expects SVE2.1 or SME2 target and a predicate register");
#ifdef EXPENSIVE_CHECKS
assert(!(PPRBegin < ZPRBegin) &&
@@ -3732,7 +3741,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF,
SavedRegs.test(CSRegs[i ^ 1]));
}
- if (HasPairZReg && enableMultiVectorSpillFill(Subtarget)) {
+ if (HasPairZReg && enableMultiVectorSpillFill(Subtarget, MF)) {
AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
// Find a suitable predicate register for the multi-vector spill/fill
// instructions.
diff --git a/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll b/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
index 9ba6a9db4cef4d..6556302e5d88d2 100644
--- a/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
+++ b/llvm/test/CodeGen/AArch64/sme-callee-save-restore-pairs.ll
@@ -384,40 +384,54 @@ define void @ffloat(<vscale x 4 x i32> %v) #2 {
; PAIR-NEXT: cntd x9
; PAIR-NEXT: str x9, [sp, #24] // 8-byte Folded Spill
; PAIR-NEXT: addsvl sp, sp, #-18
-; PAIR-NEXT: str p8, [sp, #11, mul vl] // 2-byte Folded Spill
-; PAIR-NEXT: ptrue pn8.b
; PAIR-NEXT: str p15, [sp, #4, mul vl] // 2-byte Folded Spill
-; PAIR-NEXT: st1b { z22.b, z23.b }, pn8, [sp, #4, mul vl] // 32-byte Folded Spill
-; PAIR-NEXT: st1b { z20.b, z21.b }, pn8, [sp, #8, mul vl] // 32-byte Folded Spill
; PAIR-NEXT: str p14, [sp, #5, mul vl] // 2-byte Folded Spill
-; PAIR-NEXT: st1b { z18.b, z19.b }, pn8, [sp, #12, mul vl] // 32-byte Folded Spill
-; PAIR-NEXT: st1b { z16.b, z17.b }, pn8, [sp, #16, mul vl] // 32-byte Folded Spill
; PAIR-NEXT: str p13, [sp, #6, mul vl] // 2-byte Folded Spill
-; PAIR-NEXT: st1b { z14.b, z15.b }, pn8, [sp, #20, mul vl] // 32-byte Folded Spill
-; PAIR-NEXT: st1b { z12.b, z13.b }, pn8, [sp, #24, mul vl] // 32-byte Folded Spill
; PAIR-NEXT: str p12, [sp, #7, mul vl] // 2-byte Folded Spill
-; PAIR-NEXT: st1b { z10.b, z11.b }, pn8, [sp, #28, mul vl] // 32-byte Folded Spill
-; PAIR-NEXT: st1b { z8.b, z9.b }, pn8, [sp, #32, mul vl] // 32-byte Folded Spill
; PAIR-NEXT: str p11, [sp, #8, mul vl] // 2-byte Folded Spill
; PAIR-NEXT: str p10, [sp, #9, mul vl] // 2-byte Folded Spill
; PAIR-NEXT: str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str p8, [sp, #11, mul vl] // 2-byte Folded Spill
; PAIR-NEXT: str p7, [sp, #12, mul vl] // 2-byte Folded Spill
; PAIR-NEXT: str p6, [sp, #13, mul vl] // 2-byte Folded Spill
; PAIR-NEXT: str p5, [sp, #14, mul vl] // 2-byte Folded Spill
; PAIR-NEXT: str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; PAIR-NEXT: str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; PAIR-NEXT: str z8, [sp, #17, mul vl] // 16-byte Folded Spill
; PAIR-NEXT: smstart sm
; PAIR-NEXT: smstop sm
; PAIR-NEXT: bl my_func
-; PAIR-NEXT: ptrue pn8.b
+; PAIR-NEXT: ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; PAIR-NEXT: ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
; PAIR-NEXT: ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
-; PAIR-NEXT: ld1b { z22.b, z23.b }, pn8/z, [sp, #4, mul vl] // 32-byte Folded Reload
-; PAIR-NEXT: ld1b { z20.b, z21.b }, pn8/z, [sp, #8, mul vl] // 32-byte Folded Reload
-; PAIR-NEXT: ld1b { z18.b, z19.b }, pn8/z, [sp, #12, mul vl] // 32-byte Folded Reload
-; PAIR-NEXT: ld1b { z16.b, z17.b }, pn8/z, [sp, #16, mul vl] // 32-byte Folded Reload
-; PAIR-NEXT: ld1b { z14.b, z15.b }, pn8/z, [sp, #20, mul vl] // 32-byte Folded Reload
-; PAIR-NEXT: ld1b { z12.b, z13.b }, pn8/z, [sp, #24, mul vl] // 32-byte Folded Reload
-; PAIR-NEXT: ld1b { z10.b, z11.b }, pn8/z, [sp, #28, mul vl] // 32-byte Folded Reload
-; PAIR-NEXT: ld1b { z8.b, z9.b }, pn8/z, [sp, #32, mul vl] // 32-byte Folded Reload
; PAIR-NEXT: ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
; PAIR-NEXT: ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
; PAIR-NEXT: ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
More information about the llvm-commits
mailing list