[llvm] [LLVM][CodeGen][SVE] Lower to multivector stores (PR #207397)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 01:56:24 PDT 2026
https://github.com/jacob-crawley updated https://github.com/llvm/llvm-project/pull/207397
>From 6fb9bfb6c7c449ee123844cd837f8b0939d7a7ee Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 2 Jul 2026 15:40:29 +0000
Subject: [PATCH 1/7] [LLVM][CodeGen][SVE] Lower to multivector stores instead
of splitting them.
Lowers unpredicated stores of scalable vectors that are two or four
times the width of a legal SVE type to multi-vector operations when
subregister liveness is enabled.
This matches the existing approach for multi-vector load lowering.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 110 ++++
.../CodeGen/AArch64/sve-multivector-loads.ll | 610 ++++++++----------
2 files changed, 383 insertions(+), 337 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e883c8bb5e96e..de51972b202ea 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2106,6 +2106,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::nxv4f64, Custom);
setOperationAction(ISD::LOAD, MVT::nxv16bf16, Custom);
+ // 2x stores
+ setOperationAction(ISD::STORE, MVT::nxv32i8, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv16i16, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv8i32, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv4i64, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv16f16, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv8f32, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv4f64, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv16bf16, Custom);
+
// 4x loads
setOperationAction(ISD::LOAD, MVT::nxv64i8, Custom);
setOperationAction(ISD::LOAD, MVT::nxv32i16, Custom);
@@ -2115,6 +2125,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::nxv16f32, Custom);
setOperationAction(ISD::LOAD, MVT::nxv8f64, Custom);
setOperationAction(ISD::LOAD, MVT::nxv32bf16, Custom);
+
+ // 4x stores
+ setOperationAction(ISD::STORE, MVT::nxv64i8, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv32i16, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv16i32, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv8i64, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv32f16, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv16f32, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv8f64, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv32bf16, Custom);
}
}
@@ -7733,6 +7753,93 @@ static SDValue LowerNTStore(StoreSDNode *StoreNode, EVT VT, EVT MemVT,
return SDValue();
}
+// Lower scalable vectors that are 2/4 times the width of a legal SVE type to
+// multi-vector operations.
+static SDValue tryLowerMultiVectorStore(StoreSDNode *StoreNode,
+ SelectionDAG &DAG) {
+ SDValue Value = StoreNode->getValue();
+ EVT VT = Value.getValueType();
+ EVT MemVT = StoreNode->getMemoryVT();
+
+ if (!StoreNode->isSimple() || !StoreNode->isUnindexed() ||
+ StoreNode->isNonTemporal() || !StoreNode->getOffset().isUndef() ||
+ !VT.isScalableVector() || !VT.isSimple() || VT != MemVT)
+ return SDValue();
+
+ MVT StoreVT = VT.getSimpleVT();
+ MVT RegVT;
+ unsigned IntID;
+ unsigned NumVecs;
+
+ switch (StoreVT.SimpleTy) {
+ default:
+ return SDValue();
+
+ case MVT::nxv32i8:
+ case MVT::nxv16i16:
+ case MVT::nxv8i32:
+ case MVT::nxv4i64:
+ case MVT::nxv16f16:
+ case MVT::nxv8f32:
+ case MVT::nxv4f64:
+ case MVT::nxv16bf16:
+ IntID = Intrinsic::aarch64_sve_st1_pn_x2;
+ NumVecs = 2;
+ RegVT = StoreVT.getHalfNumVectorElementsVT();
+ break;
+ case MVT::nxv64i8:
+ case MVT::nxv32i16:
+ case MVT::nxv16i32:
+ case MVT::nxv8i64:
+ case MVT::nxv32f16:
+ case MVT::nxv16f32:
+ case MVT::nxv8f64:
+ case MVT::nxv32bf16:
+ IntID = Intrinsic::aarch64_sve_st1_pn_x4;
+ NumVecs = 4;
+ RegVT = StoreVT.getHalfNumVectorElementsVT().getHalfNumVectorElementsVT();
+ break;
+ }
+
+ unsigned PredIntID;
+ switch (StoreVT.getScalarSizeInBits()) {
+ default:
+ llvm_unreachable("covered by previous switch");
+ case 8:
+ PredIntID = Intrinsic::aarch64_sve_ptrue_c8;
+ break;
+ case 16:
+ PredIntID = Intrinsic::aarch64_sve_ptrue_c16;
+ break;
+ case 32:
+ PredIntID = Intrinsic::aarch64_sve_ptrue_c32;
+ break;
+ case 64:
+ PredIntID = Intrinsic::aarch64_sve_ptrue_c64;
+ break;
+ }
+
+ SDLoc DL(StoreNode);
+ SDValue PNg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
+ DAG.getConstant(PredIntID, DL, MVT::i64));
+
+ SmallVector<SDValue, 8> Ops;
+ Ops.push_back(StoreNode->getChain());
+ Ops.push_back(DAG.getConstant(IntID, DL, MVT::i64));
+
+ unsigned RegElts = RegVT.getVectorMinNumElements();
+ for (unsigned i = 0; i != NumVecs; ++i)
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, RegVT, Value,
+ DAG.getVectorIdxConstant(i * RegElts, DL)));
+
+ Ops.push_back(PNg);
+ Ops.push_back(StoreNode->getBasePtr());
+
+ return DAG.getMemIntrinsicNode(
+ ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
+ StoreNode->getMemoryVT(), StoreNode->getMemOperand());
+}
+
// Custom lowering for any store, vector or scalar and/or default or with
// a truncate operations. Currently only custom lower truncate operation
// from vector v4i16 to v4i8 or volatile stores of i128.
@@ -7753,6 +7860,9 @@ SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
}
if (VT.isVector()) {
+ if (SDValue Store = tryLowerMultiVectorStore(StoreNode, DAG))
+ return Store;
+
if (useSVEForFixedLengthVectorVT(
VT,
/*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors()))
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll b/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
index 985326f26a2d5..e7a4ee355f58f 100644
--- a/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
@@ -22,18 +22,18 @@ define void @load_2x_vectors_i8_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0]
; SVE2p1-SL-NEXT: add z1.b, z1.b, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.b, z0.b, #5 // =0x5
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1b { z0.b, z1.b }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i8_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x0]
-; SME2-NEXT: add z24.b, z24.b, #5 // =0x5
-; SME2-NEXT: add z16.b, z16.b, #5 // =0x5
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.b, z1.b, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.b, z0.b, #5 // =0x5
+; SME2-NEXT: st1b { z0.b, z1.b }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 32 x i8>, ptr %addr
%b = add <vscale x 32 x i8> %a, splat (i8 5)
@@ -57,25 +57,23 @@ define void @load_2x_vectors_i8_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_2x_vectors_i8_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.b
-; SVE2p1-SL-NEXT: ptrue p0.b
; SVE2p1-SL-NEXT: add x8, x0, x1
; SVE2p1-SL-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0, x1]
-; SVE2p1-SL-NEXT: add z0.b, z0.b, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.b, z1.b, #5 // =0x5
-; SVE2p1-SL-NEXT: st1b { z0.b }, p0, [x0, x1]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.b, z0.b, #5 // =0x5
+; SVE2p1-SL-NEXT: st1b { z0.b, z1.b }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i8_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
-; SME2-NEXT: ptrue p0.b
; SME2-NEXT: add x8, x0, x1
; SME2-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x0, x1]
-; SME2-NEXT: add z16.b, z16.b, #5 // =0x5
-; SME2-NEXT: add z24.b, z24.b, #5 // =0x5
-; SME2-NEXT: st1b { z16.b }, p0, [x0, x1]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.b, z1.b, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.b, z0.b, #5 // =0x5
+; SME2-NEXT: st1b { z0.b, z1.b }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i8, ptr %base, i64 %idx
%a = load <vscale x 32 x i8>, ptr %addr
@@ -101,18 +99,18 @@ define void @load_2x_vectors_i16_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0]
; SVE2p1-SL-NEXT: add z1.h, z1.h, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.h, z0.h, #5 // =0x5
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i16_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NEXT: add z24.h, z24.h, #5 // =0x5
-; SME2-NEXT: add z16.h, z16.h, #5 // =0x5
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.h, z1.h, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.h, z0.h, #5 // =0x5
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 16 x i16>, ptr %addr
%b = add <vscale x 16 x i16> %a, splat (i16 5)
@@ -136,25 +134,23 @@ define void @load_2x_vectors_i16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_2x_vectors_i16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
-; SVE2p1-SL-NEXT: ptrue p0.h
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: add z0.h, z0.h, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.h, z1.h, #5 // =0x5
-; SVE2p1-SL-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.h, z0.h, #5 // =0x5
+; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i16_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
-; SME2-NEXT: ptrue p0.h
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: add z16.h, z16.h, #5 // =0x5
-; SME2-NEXT: add z24.h, z24.h, #5 // =0x5
-; SME2-NEXT: st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.h, z1.h, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.h, z0.h, #5 // =0x5
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i16, ptr %base, i64 %idx
%a = load <vscale x 16 x i16>, ptr %addr
@@ -180,18 +176,18 @@ define void @load_2x_vectors_i32_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0]
; SVE2p1-SL-NEXT: add z1.s, z1.s, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.s, z0.s, #5 // =0x5
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i32_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0]
-; SME2-NEXT: add z24.s, z24.s, #5 // =0x5
-; SME2-NEXT: add z16.s, z16.s, #5 // =0x5
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.s, z1.s, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.s, z0.s, #5 // =0x5
+; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 8 x i32>, ptr %addr
%b = add <vscale x 8 x i32> %a, splat (i32 5)
@@ -215,25 +211,23 @@ define void @load_2x_vectors_i32_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_2x_vectors_i32_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
-; SVE2p1-SL-NEXT: ptrue p0.s
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #2
; SVE2p1-SL-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: add z0.s, z0.s, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.s, z1.s, #5 // =0x5
-; SVE2p1-SL-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.s, z0.s, #5 // =0x5
+; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i32_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
-; SME2-NEXT: ptrue p0.s
; SME2-NEXT: add x8, x0, x1, lsl #2
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT: add z16.s, z16.s, #5 // =0x5
-; SME2-NEXT: add z24.s, z24.s, #5 // =0x5
-; SME2-NEXT: st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.s, z1.s, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.s, z0.s, #5 // =0x5
+; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i32, ptr %base, i64 %idx
%a = load <vscale x 8 x i32>, ptr %addr
@@ -259,18 +253,18 @@ define void @load_2x_vectors_i64_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0]
; SVE2p1-SL-NEXT: add z1.d, z1.d, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.d, z0.d, #5 // =0x5
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i64_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0]
-; SME2-NEXT: add z24.d, z24.d, #5 // =0x5
-; SME2-NEXT: add z16.d, z16.d, #5 // =0x5
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.d, z1.d, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.d, z0.d, #5 // =0x5
+; SME2-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 4 x i64>, ptr %addr
%b = add <vscale x 4 x i64> %a, splat (i64 5)
@@ -294,25 +288,23 @@ define void @load_2x_vectors_i64_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_2x_vectors_i64_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
-; SVE2p1-SL-NEXT: ptrue p0.d
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #3
; SVE2p1-SL-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT: add z0.d, z0.d, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.d, z1.d, #5 // =0x5
-; SVE2p1-SL-NEXT: st1d { z0.d }, p0, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.d, z0.d, #5 // =0x5
+; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i64_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
-; SME2-NEXT: ptrue p0.d
; SME2-NEXT: add x8, x0, x1, lsl #3
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT: add z16.d, z16.d, #5 // =0x5
-; SME2-NEXT: add z24.d, z24.d, #5 // =0x5
-; SME2-NEXT: st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.d, z1.d, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.d, z0.d, #5 // =0x5
+; SME2-NEXT: st1d { z0.d, z1.d }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i64, ptr %base, i64 %idx
%a = load <vscale x 4 x i64>, ptr %addr
@@ -340,8 +332,7 @@ define void @load_2x_vectors_f16_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0]
; SVE2p1-SL-NEXT: fadd z1.h, p0/m, z1.h, #1.0
; SVE2p1-SL-NEXT: fadd z0.h, p0/m, z0.h, #1.0
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f16_r:
@@ -349,10 +340,11 @@ define void @load_2x_vectors_f16_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ptrue p0.h
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NEXT: fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NEXT: fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: fadd z1.h, p0/m, z1.h, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.h, p0/m, z0.h, #1.0
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 16 x half>, ptr %addr
%b = fadd <vscale x 16 x half> %a, splat (half 1.0)
@@ -379,10 +371,9 @@ define void @load_2x_vectors_f16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: ptrue p0.h
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: fadd z0.h, p0/m, z0.h, #1.0
; SVE2p1-SL-NEXT: fadd z1.h, p0/m, z1.h, #1.0
-; SVE2p1-SL-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: fadd z0.h, p0/m, z0.h, #1.0
+; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f16_rr:
@@ -391,10 +382,11 @@ define void @load_2x_vectors_f16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.h
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NEXT: fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NEXT: st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: fadd z1.h, p0/m, z1.h, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.h, p0/m, z0.h, #1.0
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr half, ptr %base, i64 %idx
%a = load <vscale x 16 x half>, ptr %addr
@@ -422,8 +414,7 @@ define void @load_2x_vectors_f32_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0]
; SVE2p1-SL-NEXT: fadd z1.s, p0/m, z1.s, #1.0
; SVE2p1-SL-NEXT: fadd z0.s, p0/m, z0.s, #1.0
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f32_r:
@@ -431,10 +422,11 @@ define void @load_2x_vectors_f32_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ptrue p0.s
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0]
-; SME2-NEXT: fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NEXT: fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: fadd z1.s, p0/m, z1.s, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.s, p0/m, z0.s, #1.0
+; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 8 x float>, ptr %addr
%b = fadd <vscale x 8 x float> %a, splat (float 1.0)
@@ -461,10 +453,9 @@ define void @load_2x_vectors_f32_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: ptrue p0.s
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #2
; SVE2p1-SL-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: fadd z0.s, p0/m, z0.s, #1.0
; SVE2p1-SL-NEXT: fadd z1.s, p0/m, z1.s, #1.0
-; SVE2p1-SL-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: fadd z0.s, p0/m, z0.s, #1.0
+; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f32_rr:
@@ -473,10 +464,11 @@ define void @load_2x_vectors_f32_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.s
; SME2-NEXT: add x8, x0, x1, lsl #2
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT: fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NEXT: fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NEXT: st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: fadd z1.s, p0/m, z1.s, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.s, p0/m, z0.s, #1.0
+; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr float, ptr %base, i64 %idx
%a = load <vscale x 8 x float>, ptr %addr
@@ -504,8 +496,7 @@ define void @load_2x_vectors_f64_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0]
; SVE2p1-SL-NEXT: fadd z1.d, p0/m, z1.d, #1.0
; SVE2p1-SL-NEXT: fadd z0.d, p0/m, z0.d, #1.0
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f64_r:
@@ -513,10 +504,11 @@ define void @load_2x_vectors_f64_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ptrue p0.d
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0]
-; SME2-NEXT: fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NEXT: fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: fadd z1.d, p0/m, z1.d, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.d, p0/m, z0.d, #1.0
+; SME2-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 4 x double>, ptr %addr
%b = fadd <vscale x 4 x double> %a, splat (double 1.0)
@@ -543,10 +535,9 @@ define void @load_2x_vectors_f64_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: ptrue p0.d
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #3
; SVE2p1-SL-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT: fadd z0.d, p0/m, z0.d, #1.0
; SVE2p1-SL-NEXT: fadd z1.d, p0/m, z1.d, #1.0
-; SVE2p1-SL-NEXT: st1d { z0.d }, p0, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: fadd z0.d, p0/m, z0.d, #1.0
+; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f64_rr:
@@ -555,10 +546,11 @@ define void @load_2x_vectors_f64_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.d
; SME2-NEXT: add x8, x0, x1, lsl #3
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT: fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NEXT: fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NEXT: st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: fadd z1.d, p0/m, z1.d, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.d, p0/m, z0.d, #1.0
+; SME2-NEXT: st1d { z0.d, z1.d }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr double, ptr %base, i64 %idx
%a = load <vscale x 4 x double>, ptr %addr
@@ -582,12 +574,11 @@ define void @load_2x_vectors_bf16_r(ptr %addr) {
; SVE2p1-SL-LABEL: load_2x_vectors_bf16_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
-; SVE2p1-SL-NEXT: fmov z0.h, #1.87500000
-; SVE2p1-SL-NEXT: ld1h { z2.h, z3.h }, pn8/z, [x0]
-; SVE2p1-SL-NEXT: bfadd z1.h, z3.h, z0.h
-; SVE2p1-SL-NEXT: bfadd z0.h, z2.h, z0.h
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: fmov z2.h, #1.87500000
+; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0]
+; SVE2p1-SL-NEXT: bfadd z1.h, z1.h, z2.h
+; SVE2p1-SL-NEXT: bfadd z0.h, z0.h, z2.h
+; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_bf16_r:
@@ -597,8 +588,7 @@ define void @load_2x_vectors_bf16_r(ptr %addr) {
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
; SME2-NEXT: bfadd z1.h, z24.h, z0.h
; SME2-NEXT: bfadd z0.h, z16.h, z0.h
-; SME2-NEXT: str z1, [x0, #1, mul vl]
-; SME2-NEXT: str z0, [x0]
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 16 x bfloat>, ptr %addr
%b = fadd <vscale x 16 x bfloat> %a, splat (bfloat 1.0)
@@ -623,14 +613,12 @@ define void @load_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_2x_vectors_bf16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
-; SVE2p1-SL-NEXT: fmov z0.h, #1.87500000
+; SVE2p1-SL-NEXT: fmov z2.h, #1.87500000
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #1
-; SVE2p1-SL-NEXT: ld1h { z2.h, z3.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: ptrue p0.h
-; SVE2p1-SL-NEXT: bfadd z1.h, z2.h, z0.h
-; SVE2p1-SL-NEXT: bfadd z0.h, z3.h, z0.h
-; SVE2p1-SL-NEXT: st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: str z0, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]
+; SVE2p1-SL-NEXT: bfadd z1.h, z1.h, z2.h
+; SVE2p1-SL-NEXT: bfadd z0.h, z0.h, z2.h
+; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_bf16_rr:
@@ -639,11 +627,9 @@ define void @load_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: fmov z0.h, #1.87500000
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: ptrue p0.h
-; SME2-NEXT: bfadd z1.h, z16.h, z0.h
-; SME2-NEXT: bfadd z0.h, z24.h, z0.h
-; SME2-NEXT: st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT: str z0, [x8, #1, mul vl]
+; SME2-NEXT: bfadd z1.h, z24.h, z0.h
+; SME2-NEXT: bfadd z0.h, z16.h, z0.h
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr bfloat, ptr %base, i64 %idx
%a = load <vscale x 16 x bfloat>, ptr %addr
@@ -679,24 +665,22 @@ define void @load_4x_vectors_i8_r(ptr %addr) {
; SVE2p1-SL-NEXT: add z2.b, z2.b, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.b, z1.b, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.b, z0.b, #5 // =0x5
-; SVE2p1-SL-NEXT: str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1b { z0.b - z3.b }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_i8_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
-; SME2-NEXT: add z28.b, z28.b, #5 // =0x5
-; SME2-NEXT: add z24.b, z24.b, #5 // =0x5
-; SME2-NEXT: add z20.b, z20.b, #5 // =0x5
-; SME2-NEXT: add z16.b, z16.b, #5 // =0x5
-; SME2-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: add z3.b, z3.b, #5 // =0x5
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: add z2.b, z2.b, #5 // =0x5
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: add z1.b, z1.b, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.b, z0.b, #5 // =0x5
+; SME2-NEXT: st1b { z0.b - z3.b }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 64 x i8>, ptr %addr
%b = add <vscale x 64 x i8> %a, splat (i8 5)
@@ -726,33 +710,29 @@ define void @load_4x_vectors_i8_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_4x_vectors_i8_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.b
-; SVE2p1-SL-NEXT: ptrue p0.b
; SVE2p1-SL-NEXT: add x8, x0, x1
; SVE2p1-SL-NEXT: ld1b { z0.b - z3.b }, pn8/z, [x0, x1]
-; SVE2p1-SL-NEXT: add z0.b, z0.b, #5 // =0x5
; SVE2p1-SL-NEXT: add z3.b, z3.b, #5 // =0x5
; SVE2p1-SL-NEXT: add z2.b, z2.b, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.b, z1.b, #5 // =0x5
-; SVE2p1-SL-NEXT: st1b { z0.b }, p0, [x0, x1]
-; SVE2p1-SL-NEXT: str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.b, z0.b, #5 // =0x5
+; SVE2p1-SL-NEXT: st1b { z0.b - z3.b }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_i8_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
-; SME2-NEXT: ptrue p0.b
; SME2-NEXT: add x8, x0, x1
; SME2-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0, x1]
-; SME2-NEXT: add z16.b, z16.b, #5 // =0x5
-; SME2-NEXT: add z28.b, z28.b, #5 // =0x5
-; SME2-NEXT: add z24.b, z24.b, #5 // =0x5
-; SME2-NEXT: add z20.b, z20.b, #5 // =0x5
-; SME2-NEXT: st1b { z16.b }, p0, [x0, x1]
-; SME2-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NEXT: str z20, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: add z3.b, z3.b, #5 // =0x5
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: add z2.b, z2.b, #5 // =0x5
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: add z1.b, z1.b, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.b, z0.b, #5 // =0x5
+; SME2-NEXT: st1b { z0.b - z3.b }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i8, ptr %base, i64 %idx
%a = load <vscale x 64 x i8>, ptr %addr
@@ -786,24 +766,22 @@ define void @load_4x_vectors_i16_r(ptr %addr) {
; SVE2p1-SL-NEXT: add z2.h, z2.h, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.h, z1.h, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.h, z0.h, #5 // =0x5
-; SVE2p1-SL-NEXT: str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_i16_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NEXT: add z28.h, z28.h, #5 // =0x5
-; SME2-NEXT: add z24.h, z24.h, #5 // =0x5
-; SME2-NEXT: add z20.h, z20.h, #5 // =0x5
-; SME2-NEXT: add z16.h, z16.h, #5 // =0x5
-; SME2-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: add z3.h, z3.h, #5 // =0x5
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: add z2.h, z2.h, #5 // =0x5
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: add z1.h, z1.h, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.h, z0.h, #5 // =0x5
+; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 32 x i16>, ptr %addr
%b = add <vscale x 32 x i16> %a, splat (i16 5)
@@ -833,33 +811,29 @@ define void @load_4x_vectors_i16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_4x_vectors_i16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
-; SVE2p1-SL-NEXT: ptrue p0.h
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-SL-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: add z0.h, z0.h, #5 // =0x5
; SVE2p1-SL-NEXT: add z3.h, z3.h, #5 // =0x5
; SVE2p1-SL-NEXT: add z2.h, z2.h, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.h, z1.h, #5 // =0x5
-; SVE2p1-SL-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.h, z0.h, #5 // =0x5
+; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_i16_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
-; SME2-NEXT: ptrue p0.h
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: add z16.h, z16.h, #5 // =0x5
-; SME2-NEXT: add z28.h, z28.h, #5 // =0x5
-; SME2-NEXT: add z24.h, z24.h, #5 // =0x5
-; SME2-NEXT: add z20.h, z20.h, #5 // =0x5
-; SME2-NEXT: st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NEXT: str z20, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: add z3.h, z3.h, #5 // =0x5
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: add z2.h, z2.h, #5 // =0x5
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: add z1.h, z1.h, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.h, z0.h, #5 // =0x5
+; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i16, ptr %base, i64 %idx
%a = load <vscale x 32 x i16>, ptr %addr
@@ -893,24 +867,22 @@ define void @load_4x_vectors_i32_r(ptr %addr) {
; SVE2p1-SL-NEXT: add z2.s, z2.s, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.s, z1.s, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.s, z0.s, #5 // =0x5
-; SVE2p1-SL-NEXT: str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1w { z0.s - z3.s }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_i32_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0]
-; SME2-NEXT: add z28.s, z28.s, #5 // =0x5
-; SME2-NEXT: add z24.s, z24.s, #5 // =0x5
-; SME2-NEXT: add z20.s, z20.s, #5 // =0x5
-; SME2-NEXT: add z16.s, z16.s, #5 // =0x5
-; SME2-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: add z3.s, z3.s, #5 // =0x5
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: add z2.s, z2.s, #5 // =0x5
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: add z1.s, z1.s, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.s, z0.s, #5 // =0x5
+; SME2-NEXT: st1w { z0.s - z3.s }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 16 x i32>, ptr %addr
%b = add <vscale x 16 x i32> %a, splat (i32 5)
@@ -940,33 +912,29 @@ define void @load_4x_vectors_i32_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_4x_vectors_i32_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
-; SVE2p1-SL-NEXT: ptrue p0.s
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #2
; SVE2p1-SL-NEXT: ld1w { z0.s - z3.s }, pn8/z, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: add z0.s, z0.s, #5 // =0x5
; SVE2p1-SL-NEXT: add z3.s, z3.s, #5 // =0x5
; SVE2p1-SL-NEXT: add z2.s, z2.s, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.s, z1.s, #5 // =0x5
-; SVE2p1-SL-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.s, z0.s, #5 // =0x5
+; SVE2p1-SL-NEXT: st1w { z0.s - z3.s }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_i32_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
-; SME2-NEXT: ptrue p0.s
; SME2-NEXT: add x8, x0, x1, lsl #2
; SME2-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT: add z16.s, z16.s, #5 // =0x5
-; SME2-NEXT: add z28.s, z28.s, #5 // =0x5
-; SME2-NEXT: add z24.s, z24.s, #5 // =0x5
-; SME2-NEXT: add z20.s, z20.s, #5 // =0x5
-; SME2-NEXT: st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NEXT: str z20, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: add z3.s, z3.s, #5 // =0x5
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: add z2.s, z2.s, #5 // =0x5
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: add z1.s, z1.s, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.s, z0.s, #5 // =0x5
+; SME2-NEXT: st1w { z0.s - z3.s }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i32, ptr %base, i64 %idx
%a = load <vscale x 16 x i32>, ptr %addr
@@ -1000,24 +968,22 @@ define void @load_4x_vectors_i64_r(ptr %addr) {
; SVE2p1-SL-NEXT: add z2.d, z2.d, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.d, z1.d, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.d, z0.d, #5 // =0x5
-; SVE2p1-SL-NEXT: str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1d { z0.d - z3.d }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_i64_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0]
-; SME2-NEXT: add z28.d, z28.d, #5 // =0x5
-; SME2-NEXT: add z24.d, z24.d, #5 // =0x5
-; SME2-NEXT: add z20.d, z20.d, #5 // =0x5
-; SME2-NEXT: add z16.d, z16.d, #5 // =0x5
-; SME2-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: add z3.d, z3.d, #5 // =0x5
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: add z2.d, z2.d, #5 // =0x5
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: add z1.d, z1.d, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.d, z0.d, #5 // =0x5
+; SME2-NEXT: st1d { z0.d - z3.d }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 8 x i64>, ptr %addr
%b = add <vscale x 8 x i64> %a, splat (i64 5)
@@ -1047,33 +1013,29 @@ define void @load_4x_vectors_i64_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_4x_vectors_i64_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
-; SVE2p1-SL-NEXT: ptrue p0.d
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #3
; SVE2p1-SL-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT: add z0.d, z0.d, #5 // =0x5
; SVE2p1-SL-NEXT: add z3.d, z3.d, #5 // =0x5
; SVE2p1-SL-NEXT: add z2.d, z2.d, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.d, z1.d, #5 // =0x5
-; SVE2p1-SL-NEXT: st1d { z0.d }, p0, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT: str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.d, z0.d, #5 // =0x5
+; SVE2p1-SL-NEXT: st1d { z0.d - z3.d }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_i64_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
-; SME2-NEXT: ptrue p0.d
; SME2-NEXT: add x8, x0, x1, lsl #3
; SME2-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT: add z16.d, z16.d, #5 // =0x5
-; SME2-NEXT: add z28.d, z28.d, #5 // =0x5
-; SME2-NEXT: add z24.d, z24.d, #5 // =0x5
-; SME2-NEXT: add z20.d, z20.d, #5 // =0x5
-; SME2-NEXT: st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NEXT: str z20, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: add z3.d, z3.d, #5 // =0x5
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: add z2.d, z2.d, #5 // =0x5
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: add z1.d, z1.d, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.d, z0.d, #5 // =0x5
+; SME2-NEXT: st1d { z0.d - z3.d }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i64, ptr %base, i64 %idx
%a = load <vscale x 8 x i64>, ptr %addr
@@ -1109,10 +1071,7 @@ define void @load_4x_vectors_f16_r(ptr %addr) {
; SVE2p1-SL-NEXT: fadd z2.h, p0/m, z2.h, #1.0
; SVE2p1-SL-NEXT: fadd z1.h, p0/m, z1.h, #1.0
; SVE2p1-SL-NEXT: fadd z0.h, p0/m, z0.h, #1.0
-; SVE2p1-SL-NEXT: str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_f16_r:
@@ -1120,14 +1079,15 @@ define void @load_4x_vectors_f16_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ptrue p0.h
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NEXT: fadd z28.h, p0/m, z28.h, #1.0
-; SME2-NEXT: fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NEXT: fadd z20.h, p0/m, z20.h, #1.0
-; SME2-NEXT: fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: fadd z3.h, p0/m, z3.h, #1.0
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: fadd z2.h, p0/m, z2.h, #1.0
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: fadd z1.h, p0/m, z1.h, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.h, p0/m, z0.h, #1.0
+; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 32 x half>, ptr %addr
%b = fadd <vscale x 32 x half> %a, splat (half 1.0)
@@ -1160,14 +1120,11 @@ define void @load_4x_vectors_f16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: ptrue p0.h
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-SL-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: fadd z0.h, p0/m, z0.h, #1.0
; SVE2p1-SL-NEXT: fadd z3.h, p0/m, z3.h, #1.0
; SVE2p1-SL-NEXT: fadd z2.h, p0/m, z2.h, #1.0
; SVE2p1-SL-NEXT: fadd z1.h, p0/m, z1.h, #1.0
-; SVE2p1-SL-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: fadd z0.h, p0/m, z0.h, #1.0
+; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_f16_rr:
@@ -1176,14 +1133,15 @@ define void @load_4x_vectors_f16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.h
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NEXT: fadd z28.h, p0/m, z28.h, #1.0
-; SME2-NEXT: fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NEXT: fadd z20.h, p0/m, z20.h, #1.0
-; SME2-NEXT: st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NEXT: str z20, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: fadd z3.h, p0/m, z3.h, #1.0
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: fadd z2.h, p0/m, z2.h, #1.0
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: fadd z1.h, p0/m, z1.h, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.h, p0/m, z0.h, #1.0
+; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr half, ptr %base, i64 %idx
%a = load <vscale x 32 x half>, ptr %addr
@@ -1219,10 +1177,7 @@ define void @load_4x_vectors_f32_r(ptr %addr) {
; SVE2p1-SL-NEXT: fadd z2.s, p0/m, z2.s, #1.0
; SVE2p1-SL-NEXT: fadd z1.s, p0/m, z1.s, #1.0
; SVE2p1-SL-NEXT: fadd z0.s, p0/m, z0.s, #1.0
-; SVE2p1-SL-NEXT: str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1w { z0.s - z3.s }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_f32_r:
@@ -1230,14 +1185,15 @@ define void @load_4x_vectors_f32_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ptrue p0.s
; SME2-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0]
-; SME2-NEXT: fadd z28.s, p0/m, z28.s, #1.0
-; SME2-NEXT: fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NEXT: fadd z20.s, p0/m, z20.s, #1.0
-; SME2-NEXT: fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: fadd z3.s, p0/m, z3.s, #1.0
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: fadd z2.s, p0/m, z2.s, #1.0
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: fadd z1.s, p0/m, z1.s, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.s, p0/m, z0.s, #1.0
+; SME2-NEXT: st1w { z0.s - z3.s }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 16 x float>, ptr %addr
%b = fadd <vscale x 16 x float> %a, splat (float 1.0)
@@ -1270,14 +1226,11 @@ define void @load_4x_vectors_f32_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: ptrue p0.s
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #2
; SVE2p1-SL-NEXT: ld1w { z0.s - z3.s }, pn8/z, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: fadd z0.s, p0/m, z0.s, #1.0
; SVE2p1-SL-NEXT: fadd z3.s, p0/m, z3.s, #1.0
; SVE2p1-SL-NEXT: fadd z2.s, p0/m, z2.s, #1.0
; SVE2p1-SL-NEXT: fadd z1.s, p0/m, z1.s, #1.0
-; SVE2p1-SL-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: fadd z0.s, p0/m, z0.s, #1.0
+; SVE2p1-SL-NEXT: st1w { z0.s - z3.s }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_f32_rr:
@@ -1286,14 +1239,15 @@ define void @load_4x_vectors_f32_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.s
; SME2-NEXT: add x8, x0, x1, lsl #2
; SME2-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT: fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NEXT: fadd z28.s, p0/m, z28.s, #1.0
-; SME2-NEXT: fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NEXT: fadd z20.s, p0/m, z20.s, #1.0
-; SME2-NEXT: st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NEXT: str z20, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: fadd z3.s, p0/m, z3.s, #1.0
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: fadd z2.s, p0/m, z2.s, #1.0
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: fadd z1.s, p0/m, z1.s, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.s, p0/m, z0.s, #1.0
+; SME2-NEXT: st1w { z0.s - z3.s }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr float, ptr %base, i64 %idx
%a = load <vscale x 16 x float>, ptr %addr
@@ -1329,10 +1283,7 @@ define void @load_4x_vectors_f64_r(ptr %addr) {
; SVE2p1-SL-NEXT: fadd z2.d, p0/m, z2.d, #1.0
; SVE2p1-SL-NEXT: fadd z1.d, p0/m, z1.d, #1.0
; SVE2p1-SL-NEXT: fadd z0.d, p0/m, z0.d, #1.0
-; SVE2p1-SL-NEXT: str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1d { z0.d - z3.d }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_f64_r:
@@ -1340,14 +1291,15 @@ define void @load_4x_vectors_f64_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ptrue p0.d
; SME2-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0]
-; SME2-NEXT: fadd z28.d, p0/m, z28.d, #1.0
-; SME2-NEXT: fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NEXT: fadd z20.d, p0/m, z20.d, #1.0
-; SME2-NEXT: fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NEXT: str z28, [x0, #3, mul vl]
-; SME2-NEXT: str z24, [x0, #2, mul vl]
-; SME2-NEXT: str z20, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: fadd z3.d, p0/m, z3.d, #1.0
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: fadd z2.d, p0/m, z2.d, #1.0
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: fadd z1.d, p0/m, z1.d, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.d, p0/m, z0.d, #1.0
+; SME2-NEXT: st1d { z0.d - z3.d }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 8 x double>, ptr %addr
%b = fadd <vscale x 8 x double> %a, splat (double 1.0)
@@ -1380,14 +1332,11 @@ define void @load_4x_vectors_f64_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: ptrue p0.d
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #3
; SVE2p1-SL-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT: fadd z0.d, p0/m, z0.d, #1.0
; SVE2p1-SL-NEXT: fadd z3.d, p0/m, z3.d, #1.0
; SVE2p1-SL-NEXT: fadd z2.d, p0/m, z2.d, #1.0
; SVE2p1-SL-NEXT: fadd z1.d, p0/m, z1.d, #1.0
-; SVE2p1-SL-NEXT: st1d { z0.d }, p0, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT: str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT: str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: fadd z0.d, p0/m, z0.d, #1.0
+; SVE2p1-SL-NEXT: st1d { z0.d - z3.d }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_f64_rr:
@@ -1396,14 +1345,15 @@ define void @load_4x_vectors_f64_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.d
; SME2-NEXT: add x8, x0, x1, lsl #3
; SME2-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT: fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NEXT: fadd z28.d, p0/m, z28.d, #1.0
-; SME2-NEXT: fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NEXT: fadd z20.d, p0/m, z20.d, #1.0
-; SME2-NEXT: st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NEXT: str z28, [x8, #3, mul vl]
-; SME2-NEXT: str z24, [x8, #2, mul vl]
-; SME2-NEXT: str z20, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z3, z28
+; SME2-NEXT: fadd z3.d, p0/m, z3.d, #1.0
+; SME2-NEXT: movprfx z2, z24
+; SME2-NEXT: fadd z2.d, p0/m, z2.d, #1.0
+; SME2-NEXT: movprfx z1, z20
+; SME2-NEXT: fadd z1.d, p0/m, z1.d, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.d, p0/m, z0.d, #1.0
+; SME2-NEXT: st1d { z0.d - z3.d }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr double, ptr %base, i64 %idx
%a = load <vscale x 8 x double>, ptr %addr
@@ -1433,16 +1383,13 @@ define void @load_4x_vectors_bf16_r(ptr %addr) {
; SVE2p1-SL-LABEL: load_4x_vectors_bf16_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
-; SVE2p1-SL-NEXT: fmov z0.h, #1.87500000
-; SVE2p1-SL-NEXT: ld1h { z4.h - z7.h }, pn8/z, [x0]
-; SVE2p1-SL-NEXT: bfadd z1.h, z7.h, z0.h
-; SVE2p1-SL-NEXT: bfadd z2.h, z6.h, z0.h
-; SVE2p1-SL-NEXT: str z1, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT: bfadd z1.h, z5.h, z0.h
-; SVE2p1-SL-NEXT: bfadd z0.h, z4.h, z0.h
-; SVE2p1-SL-NEXT: str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: fmov z4.h, #1.87500000
+; SVE2p1-SL-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0]
+; SVE2p1-SL-NEXT: bfadd z3.h, z3.h, z4.h
+; SVE2p1-SL-NEXT: bfadd z2.h, z2.h, z4.h
+; SVE2p1-SL-NEXT: bfadd z1.h, z1.h, z4.h
+; SVE2p1-SL-NEXT: bfadd z0.h, z0.h, z4.h
+; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_bf16_r:
@@ -1450,14 +1397,11 @@ define void @load_4x_vectors_bf16_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: fmov z0.h, #1.87500000
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NEXT: bfadd z1.h, z28.h, z0.h
+; SME2-NEXT: bfadd z3.h, z28.h, z0.h
; SME2-NEXT: bfadd z2.h, z24.h, z0.h
-; SME2-NEXT: str z1, [x0, #3, mul vl]
; SME2-NEXT: bfadd z1.h, z20.h, z0.h
; SME2-NEXT: bfadd z0.h, z16.h, z0.h
-; SME2-NEXT: str z2, [x0, #2, mul vl]
-; SME2-NEXT: str z1, [x0, #1, mul vl]
-; SME2-NEXT: str z0, [x0]
+; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 32 x bfloat>, ptr %addr
%b = fadd <vscale x 32 x bfloat> %a, splat (bfloat 1.0)
@@ -1488,18 +1432,14 @@ define void @load_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_4x_vectors_bf16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
-; SVE2p1-SL-NEXT: fmov z0.h, #1.87500000
+; SVE2p1-SL-NEXT: fmov z4.h, #1.87500000
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #1
-; SVE2p1-SL-NEXT: ld1h { z4.h - z7.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: ptrue p0.h
-; SVE2p1-SL-NEXT: bfadd z1.h, z4.h, z0.h
-; SVE2p1-SL-NEXT: bfadd z2.h, z7.h, z0.h
-; SVE2p1-SL-NEXT: st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: bfadd z1.h, z6.h, z0.h
-; SVE2p1-SL-NEXT: bfadd z0.h, z5.h, z0.h
-; SVE2p1-SL-NEXT: str z2, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT: str z1, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]
+; SVE2p1-SL-NEXT: bfadd z3.h, z3.h, z4.h
+; SVE2p1-SL-NEXT: bfadd z2.h, z2.h, z4.h
+; SVE2p1-SL-NEXT: bfadd z1.h, z1.h, z4.h
+; SVE2p1-SL-NEXT: bfadd z0.h, z0.h, z4.h
+; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_4x_vectors_bf16_rr:
@@ -1508,15 +1448,11 @@ define void @load_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: fmov z0.h, #1.87500000
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: ptrue p0.h
-; SME2-NEXT: bfadd z1.h, z16.h, z0.h
-; SME2-NEXT: bfadd z2.h, z28.h, z0.h
-; SME2-NEXT: st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT: bfadd z1.h, z24.h, z0.h
-; SME2-NEXT: bfadd z0.h, z20.h, z0.h
-; SME2-NEXT: str z2, [x8, #3, mul vl]
-; SME2-NEXT: str z1, [x8, #2, mul vl]
-; SME2-NEXT: str z0, [x8, #1, mul vl]
+; SME2-NEXT: bfadd z3.h, z28.h, z0.h
+; SME2-NEXT: bfadd z2.h, z24.h, z0.h
+; SME2-NEXT: bfadd z1.h, z20.h, z0.h
+; SME2-NEXT: bfadd z0.h, z16.h, z0.h
+; SME2-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr bfloat, ptr %base, i64 %idx
%a = load <vscale x 32 x bfloat>, ptr %addr
>From fc163d831af9f5555a8fa8f730dd81943ac4a45d Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 3 Jul 2026 14:42:32 +0000
Subject: [PATCH 2/7] update sve-vector-interleave
---
.../CodeGen/AArch64/sve-vector-interleave.ll | 204 ++++++++++++------
1 file changed, 144 insertions(+), 60 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
index 81a0cc242fc11..d5de1761e9a49 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
@@ -204,26 +204,54 @@ define <vscale x 6 x half> @interleave3_nxv6f16(<vscale x 2 x half> %vec0, <vsca
}
define <vscale x 12 x half> @interleave3_nxv12f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1, <vscale x 4 x half> %vec2) {
-; CHECK-LABEL: interleave3_nxv12f16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-5
-; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: addpl x8, sp, #4
-; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [sp]
-; CHECK-NEXT: ldr z0, [sp, #1, mul vl]
-; CHECK-NEXT: ldr z1, [sp]
-; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
-; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
-; CHECK-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
-; CHECK-NEXT: str z0, [sp, #3, mul vl]
-; CHECK-NEXT: ldr z1, [sp, #4, mul vl]
-; CHECK-NEXT: ldr z0, [sp, #3, mul vl]
-; CHECK-NEXT: addvl sp, sp, #5
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
+; SVE-LABEL: interleave3_nxv12f16:
+; SVE: // %bb.0:
+; SVE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SVE-NEXT: addvl sp, sp, #-5
+; SVE-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
+; SVE-NEXT: .cfi_offset w29, -16
+; SVE-NEXT: ptrue p0.s
+; SVE-NEXT: addpl x8, sp, #4
+; SVE-NEXT: st3w { z0.s - z2.s }, p0, [sp]
+; SVE-NEXT: ldr z0, [sp, #1, mul vl]
+; SVE-NEXT: ldr z1, [sp]
+; SVE-NEXT: ldr z2, [sp, #2, mul vl]
+; SVE-NEXT: uzp1 z0.h, z1.h, z0.h
+; SVE-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
+; SVE-NEXT: str z0, [sp, #3, mul vl]
+; SVE-NEXT: ldr z1, [sp, #4, mul vl]
+; SVE-NEXT: ldr z0, [sp, #3, mul vl]
+; SVE-NEXT: addvl sp, sp, #5
+; SVE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SVE-NEXT: ret
+;
+; SME2-LABEL: interleave3_nxv12f16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-5
+; SME2-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x30, 0x1e, 0x22 // sp + 16 + 48 * VG
+; SME2-NEXT: .cfi_offset w29, -16
+; SME2-NEXT: ptrue p0.s
+; SME2-NEXT: ptrue pn8.h
+; SME2-NEXT: addvl x8, sp, #1
+; SME2-NEXT: st3w { z0.s - z2.s }, p0, [sp]
+; SME2-NEXT: ldr z0, [sp, #1, mul vl]
+; SME2-NEXT: ldr z1, [sp]
+; SME2-NEXT: ldr z2, [sp, #2, mul vl]
+; SME2-NEXT: uzp1 z0.h, z1.h, z0.h
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8, #2, mul vl]
+; SME2-NEXT: addpl x8, sp, #4
+; SME2-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
+; SME2-NEXT: str z0, [sp, #3, mul vl]
+; SME2-NEXT: ldr z1, [sp, #4, mul vl]
+; SME2-NEXT: ldr z0, [sp, #3, mul vl]
+; SME2-NEXT: addvl sp, sp, #5
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
%retval = call <vscale x 12 x half> @llvm.vector.interleave3.nxv12f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1, <vscale x 4 x half> %vec2)
ret <vscale x 12 x half> %retval
}
@@ -248,26 +276,54 @@ define <vscale x 24 x half> @interleave3_nxv24f16(<vscale x 8 x half> %vec0, <vs
}
define <vscale x 6 x float> @interleave3_nxv6f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1, <vscale x 2 x float> %vec2) {
-; CHECK-LABEL: interleave3_nxv6f32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-5
-; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: addpl x8, sp, #4
-; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [sp]
-; CHECK-NEXT: ldr z0, [sp, #1, mul vl]
-; CHECK-NEXT: ldr z1, [sp]
-; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
-; CHECK-NEXT: uzp1 z0.s, z1.s, z0.s
-; CHECK-NEXT: st1w { z2.d }, p0, [x8, #7, mul vl]
-; CHECK-NEXT: str z0, [sp, #3, mul vl]
-; CHECK-NEXT: ldr z1, [sp, #4, mul vl]
-; CHECK-NEXT: ldr z0, [sp, #3, mul vl]
-; CHECK-NEXT: addvl sp, sp, #5
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
+; SVE-LABEL: interleave3_nxv6f32:
+; SVE: // %bb.0:
+; SVE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SVE-NEXT: addvl sp, sp, #-5
+; SVE-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
+; SVE-NEXT: .cfi_offset w29, -16
+; SVE-NEXT: ptrue p0.d
+; SVE-NEXT: addpl x8, sp, #4
+; SVE-NEXT: st3d { z0.d - z2.d }, p0, [sp]
+; SVE-NEXT: ldr z0, [sp, #1, mul vl]
+; SVE-NEXT: ldr z1, [sp]
+; SVE-NEXT: ldr z2, [sp, #2, mul vl]
+; SVE-NEXT: uzp1 z0.s, z1.s, z0.s
+; SVE-NEXT: st1w { z2.d }, p0, [x8, #7, mul vl]
+; SVE-NEXT: str z0, [sp, #3, mul vl]
+; SVE-NEXT: ldr z1, [sp, #4, mul vl]
+; SVE-NEXT: ldr z0, [sp, #3, mul vl]
+; SVE-NEXT: addvl sp, sp, #5
+; SVE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SVE-NEXT: ret
+;
+; SME2-LABEL: interleave3_nxv6f32:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-5
+; SME2-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x30, 0x1e, 0x22 // sp + 16 + 48 * VG
+; SME2-NEXT: .cfi_offset w29, -16
+; SME2-NEXT: ptrue p0.d
+; SME2-NEXT: ptrue pn8.s
+; SME2-NEXT: addvl x8, sp, #1
+; SME2-NEXT: st3d { z0.d - z2.d }, p0, [sp]
+; SME2-NEXT: ldr z0, [sp, #1, mul vl]
+; SME2-NEXT: ldr z1, [sp]
+; SME2-NEXT: ldr z2, [sp, #2, mul vl]
+; SME2-NEXT: uzp1 z0.s, z1.s, z0.s
+; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x8, #2, mul vl]
+; SME2-NEXT: addpl x8, sp, #4
+; SME2-NEXT: st1w { z2.d }, p0, [x8, #7, mul vl]
+; SME2-NEXT: str z0, [sp, #3, mul vl]
+; SME2-NEXT: ldr z1, [sp, #4, mul vl]
+; SME2-NEXT: ldr z0, [sp, #3, mul vl]
+; SME2-NEXT: addvl sp, sp, #5
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
%retval = call <vscale x 6 x float> @llvm.vector.interleave3.nxv6f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1, <vscale x 2 x float> %vec2)
ret <vscale x 6 x float> %retval
}
@@ -333,26 +389,54 @@ define <vscale x 6 x bfloat> @interleave3_nxv6bf16(<vscale x 2 x bfloat> %vec0,
}
define <vscale x 12 x bfloat> @interleave3_nxv12bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1, <vscale x 4 x bfloat> %vec2) {
-; CHECK-LABEL: interleave3_nxv12bf16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; CHECK-NEXT: addvl sp, sp, #-5
-; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
-; CHECK-NEXT: .cfi_offset w29, -16
-; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: addpl x8, sp, #4
-; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [sp]
-; CHECK-NEXT: ldr z0, [sp, #1, mul vl]
-; CHECK-NEXT: ldr z1, [sp]
-; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
-; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
-; CHECK-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
-; CHECK-NEXT: str z0, [sp, #3, mul vl]
-; CHECK-NEXT: ldr z1, [sp, #4, mul vl]
-; CHECK-NEXT: ldr z0, [sp, #3, mul vl]
-; CHECK-NEXT: addvl sp, sp, #5
-; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; CHECK-NEXT: ret
+; SVE-LABEL: interleave3_nxv12bf16:
+; SVE: // %bb.0:
+; SVE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SVE-NEXT: addvl sp, sp, #-5
+; SVE-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
+; SVE-NEXT: .cfi_offset w29, -16
+; SVE-NEXT: ptrue p0.s
+; SVE-NEXT: addpl x8, sp, #4
+; SVE-NEXT: st3w { z0.s - z2.s }, p0, [sp]
+; SVE-NEXT: ldr z0, [sp, #1, mul vl]
+; SVE-NEXT: ldr z1, [sp]
+; SVE-NEXT: ldr z2, [sp, #2, mul vl]
+; SVE-NEXT: uzp1 z0.h, z1.h, z0.h
+; SVE-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
+; SVE-NEXT: str z0, [sp, #3, mul vl]
+; SVE-NEXT: ldr z1, [sp, #4, mul vl]
+; SVE-NEXT: ldr z0, [sp, #3, mul vl]
+; SVE-NEXT: addvl sp, sp, #5
+; SVE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SVE-NEXT: ret
+;
+; SME2-LABEL: interleave3_nxv12bf16:
+; SME2: // %bb.0:
+; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; SME2-NEXT: addvl sp, sp, #-1
+; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
+; SME2-NEXT: addvl sp, sp, #-5
+; SME2-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x30, 0x1e, 0x22 // sp + 16 + 48 * VG
+; SME2-NEXT: .cfi_offset w29, -16
+; SME2-NEXT: ptrue p0.s
+; SME2-NEXT: ptrue pn8.h
+; SME2-NEXT: addvl x8, sp, #1
+; SME2-NEXT: st3w { z0.s - z2.s }, p0, [sp]
+; SME2-NEXT: ldr z0, [sp, #1, mul vl]
+; SME2-NEXT: ldr z1, [sp]
+; SME2-NEXT: ldr z2, [sp, #2, mul vl]
+; SME2-NEXT: uzp1 z0.h, z1.h, z0.h
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8, #2, mul vl]
+; SME2-NEXT: addpl x8, sp, #4
+; SME2-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
+; SME2-NEXT: str z0, [sp, #3, mul vl]
+; SME2-NEXT: ldr z1, [sp, #4, mul vl]
+; SME2-NEXT: ldr z0, [sp, #3, mul vl]
+; SME2-NEXT: addvl sp, sp, #5
+; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
+; SME2-NEXT: addvl sp, sp, #1
+; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; SME2-NEXT: ret
%retval = call <vscale x 12 x bfloat> @llvm.vector.interleave3.nxv12bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1, <vscale x 4 x bfloat> %vec2)
ret <vscale x 12 x bfloat> %retval
}
>From d38a28253a2c063186b98a8e05f8aacfacd5a785 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 7 Jul 2026 16:29:02 +0000
Subject: [PATCH 3/7] Put shared load/store logic into helpers
---
.../Target/AArch64/AArch64ISelLowering.cpp | 274 ++++++++----------
1 file changed, 116 insertions(+), 158 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index de51972b202ea..883fce00b6aa4 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2096,45 +2096,28 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
if (Subtarget->enableSubRegLiveness() &&
(Subtarget->hasSVE2p1() ||
(Subtarget->hasSME2() && Subtarget->isStreaming()))) {
- // 2x loads
- setOperationAction(ISD::LOAD, MVT::nxv32i8, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv16i16, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv8i32, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv4i64, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv16f16, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv8f32, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv4f64, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv16bf16, Custom);
-
- // 2x stores
- setOperationAction(ISD::STORE, MVT::nxv32i8, Custom);
- setOperationAction(ISD::STORE, MVT::nxv16i16, Custom);
- setOperationAction(ISD::STORE, MVT::nxv8i32, Custom);
- setOperationAction(ISD::STORE, MVT::nxv4i64, Custom);
- setOperationAction(ISD::STORE, MVT::nxv16f16, Custom);
- setOperationAction(ISD::STORE, MVT::nxv8f32, Custom);
- setOperationAction(ISD::STORE, MVT::nxv4f64, Custom);
- setOperationAction(ISD::STORE, MVT::nxv16bf16, Custom);
-
- // 4x loads
- setOperationAction(ISD::LOAD, MVT::nxv64i8, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv32i16, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv16i32, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv8i64, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv32f16, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv16f32, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv8f64, Custom);
- setOperationAction(ISD::LOAD, MVT::nxv32bf16, Custom);
-
- // 4x stores
- setOperationAction(ISD::STORE, MVT::nxv64i8, Custom);
- setOperationAction(ISD::STORE, MVT::nxv32i16, Custom);
- setOperationAction(ISD::STORE, MVT::nxv16i32, Custom);
- setOperationAction(ISD::STORE, MVT::nxv8i64, Custom);
- setOperationAction(ISD::STORE, MVT::nxv32f16, Custom);
- setOperationAction(ISD::STORE, MVT::nxv16f32, Custom);
- setOperationAction(ISD::STORE, MVT::nxv8f64, Custom);
- setOperationAction(ISD::STORE, MVT::nxv32bf16, Custom);
+
+ for (unsigned Opcode : {ISD::LOAD, ISD::STORE}) {
+ // 2x multi-vector load/stores
+ setOperationAction(Opcode, MVT::nxv32i8, Custom);
+ setOperationAction(Opcode, MVT::nxv16i16, Custom);
+ setOperationAction(Opcode, MVT::nxv8i32, Custom);
+ setOperationAction(Opcode, MVT::nxv4i64, Custom);
+ setOperationAction(Opcode, MVT::nxv16f16, Custom);
+ setOperationAction(Opcode, MVT::nxv8f32, Custom);
+ setOperationAction(Opcode, MVT::nxv4f64, Custom);
+ setOperationAction(Opcode, MVT::nxv16bf16, Custom);
+
+ // 4x multi-vector load/stores
+ setOperationAction(Opcode, MVT::nxv64i8, Custom);
+ setOperationAction(Opcode, MVT::nxv32i16, Custom);
+ setOperationAction(Opcode, MVT::nxv16i32, Custom);
+ setOperationAction(Opcode, MVT::nxv8i64, Custom);
+ setOperationAction(Opcode, MVT::nxv32f16, Custom);
+ setOperationAction(Opcode, MVT::nxv16f32, Custom);
+ setOperationAction(Opcode, MVT::nxv8f64, Custom);
+ setOperationAction(Opcode, MVT::nxv32bf16, Custom);
+ }
}
}
@@ -7753,27 +7736,20 @@ static SDValue LowerNTStore(StoreSDNode *StoreNode, EVT VT, EVT MemVT,
return SDValue();
}
-// Lower scalable vectors that are 2/4 times the width of a legal SVE type to
-// multi-vector operations.
-static SDValue tryLowerMultiVectorStore(StoreSDNode *StoreNode,
- SelectionDAG &DAG) {
- SDValue Value = StoreNode->getValue();
- EVT VT = Value.getValueType();
- EVT MemVT = StoreNode->getMemoryVT();
-
- if (!StoreNode->isSimple() || !StoreNode->isUnindexed() ||
- StoreNode->isNonTemporal() || !StoreNode->getOffset().isUndef() ||
- !VT.isScalableVector() || !VT.isSimple() || VT != MemVT)
- return SDValue();
-
- MVT StoreVT = VT.getSimpleVT();
+struct SVEMultiVectorInfo {
MVT RegVT;
- unsigned IntID;
unsigned NumVecs;
+ Intrinsic::ID LoadIntID;
+ Intrinsic::ID StoreIntID;
+ Intrinsic::ID PTrueIntID;
+};
+
+static std::optional<SVEMultiVectorInfo> getSVEMultiVectorInfo(MVT VT) {
+ SVEMultiVectorInfo Info;
- switch (StoreVT.SimpleTy) {
+ switch (VT.SimpleTy) {
default:
- return SDValue();
+ return std::nullopt;
case MVT::nxv32i8:
case MVT::nxv16i16:
@@ -7783,9 +7759,10 @@ static SDValue tryLowerMultiVectorStore(StoreSDNode *StoreNode,
case MVT::nxv8f32:
case MVT::nxv4f64:
case MVT::nxv16bf16:
- IntID = Intrinsic::aarch64_sve_st1_pn_x2;
- NumVecs = 2;
- RegVT = StoreVT.getHalfNumVectorElementsVT();
+ Info.LoadIntID = Intrinsic::aarch64_sve_ld1_pn_x2;
+ Info.StoreIntID = Intrinsic::aarch64_sve_st1_pn_x2;
+ Info.NumVecs = 2;
+ Info.RegVT = VT.getHalfNumVectorElementsVT();
break;
case MVT::nxv64i8:
case MVT::nxv32i16:
@@ -7795,41 +7772,68 @@ static SDValue tryLowerMultiVectorStore(StoreSDNode *StoreNode,
case MVT::nxv16f32:
case MVT::nxv8f64:
case MVT::nxv32bf16:
- IntID = Intrinsic::aarch64_sve_st1_pn_x4;
- NumVecs = 4;
- RegVT = StoreVT.getHalfNumVectorElementsVT().getHalfNumVectorElementsVT();
+ Info.LoadIntID = Intrinsic::aarch64_sve_ld1_pn_x4;
+ Info.StoreIntID = Intrinsic::aarch64_sve_st1_pn_x4;
+ Info.NumVecs = 4;
+ Info.RegVT = VT.getHalfNumVectorElementsVT().getHalfNumVectorElementsVT();
break;
}
- unsigned PredIntID;
- switch (StoreVT.getScalarSizeInBits()) {
+ switch (VT.getScalarSizeInBits()) {
default:
llvm_unreachable("covered by previous switch");
case 8:
- PredIntID = Intrinsic::aarch64_sve_ptrue_c8;
+ Info.PTrueIntID = Intrinsic::aarch64_sve_ptrue_c8;
break;
case 16:
- PredIntID = Intrinsic::aarch64_sve_ptrue_c16;
+ Info.PTrueIntID = Intrinsic::aarch64_sve_ptrue_c16;
break;
case 32:
- PredIntID = Intrinsic::aarch64_sve_ptrue_c32;
+ Info.PTrueIntID = Intrinsic::aarch64_sve_ptrue_c32;
break;
case 64:
- PredIntID = Intrinsic::aarch64_sve_ptrue_c64;
+ Info.PTrueIntID = Intrinsic::aarch64_sve_ptrue_c64;
break;
}
+ return Info;
+}
+
+static bool isValidSVEMultiVectorOp(const LSBaseSDNode *LSNode, EVT VT) {
+ return LSNode->isSimple() && LSNode->isUnindexed() &&
+ LSNode->getOffset().isUndef() && VT.isScalableVector() &&
+ VT.isSimple() && VT == LSNode->getMemoryVT();
+}
+
+// Lower scalable vectors that are 2/4 times the width of a legal SVE type to
+// multi-vector operations.
+static SDValue tryLowerMultiVectorStore(StoreSDNode *StoreNode,
+ SelectionDAG &DAG) {
+ SDValue Value = StoreNode->getValue();
+ EVT VT = Value.getValueType();
+
+ if (!isValidSVEMultiVectorOp(StoreNode, VT))
+ return SDValue();
+
+ MVT StoreVT = VT.getSimpleVT();
+ std::optional<SVEMultiVectorInfo> MultiVecInfo =
+ getSVEMultiVectorInfo(StoreVT);
+ if (!MultiVecInfo)
+ return SDValue();
+
SDLoc DL(StoreNode);
- SDValue PNg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
- DAG.getConstant(PredIntID, DL, MVT::i64));
+ SDValue PNg =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
+ DAG.getConstant(MultiVecInfo->PTrueIntID, DL, MVT::i64));
SmallVector<SDValue, 8> Ops;
Ops.push_back(StoreNode->getChain());
- Ops.push_back(DAG.getConstant(IntID, DL, MVT::i64));
+ Ops.push_back(DAG.getConstant(MultiVecInfo->StoreIntID, DL, MVT::i64));
- unsigned RegElts = RegVT.getVectorMinNumElements();
- for (unsigned i = 0; i != NumVecs; ++i)
- Ops.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, RegVT, Value,
+ unsigned RegElts = MultiVecInfo->RegVT.getVectorMinNumElements();
+ for (unsigned i = 0; i != MultiVecInfo->NumVecs; ++i)
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MultiVecInfo->RegVT,
+ Value,
DAG.getVectorIdxConstant(i * RegElts, DL)));
Ops.push_back(PNg);
@@ -7840,6 +7844,42 @@ static SDValue tryLowerMultiVectorStore(StoreSDNode *StoreNode,
StoreNode->getMemoryVT(), StoreNode->getMemOperand());
}
+static bool tryLowerMultiVectorLoad(LoadSDNode *LoadNode,
+ SmallVectorImpl<SDValue> &Results,
+ SelectionDAG &DAG) {
+ EVT VT = LoadNode->getValueType(0);
+
+ if (!isValidSVEMultiVectorOp(LoadNode, VT))
+ return false;
+
+ MVT LoadVT = VT.getSimpleVT();
+ std::optional<SVEMultiVectorInfo> MultiVecInfo =
+ getSVEMultiVectorInfo(LoadVT);
+ if (!MultiVecInfo)
+ return false;
+
+ SDLoc DL(LoadNode);
+ SDValue PNg =
+ DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
+ DAG.getConstant(MultiVecInfo->PTrueIntID, DL, MVT::i64));
+
+ SmallVector<EVT, 5> ResultVTs(MultiVecInfo->NumVecs, MultiVecInfo->RegVT);
+ ResultVTs.push_back(MVT::Other);
+
+ SDValue NewLoad =
+ DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, ResultVTs,
+ {LoadNode->getChain(),
+ DAG.getConstant(MultiVecInfo->LoadIntID, DL, MVT::i64), PNg,
+ LoadNode->getBasePtr()});
+
+ SmallVector<SDValue, 4> ResultOps;
+ for (unsigned I = 0; I != MultiVecInfo->NumVecs; ++I)
+ ResultOps.push_back(NewLoad.getValue(I));
+ Results.push_back(DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResultOps));
+ Results.push_back(NewLoad.getValue(MultiVecInfo->NumVecs) /* Chain */);
+ return true;
+}
+
// Custom lowering for any store, vector or scalar and/or default or with
// a truncate operations. Currently only custom lower truncate operation
// from vector v4i16 to v4i8 or volatile stores of i128.
@@ -31476,91 +31516,9 @@ void AArch64TargetLowering::ReplaceNodeResults(
return;
}
- LSBaseSDNode *LSNode = dyn_cast<LSBaseSDNode>(N);
- if (LSNode && LSNode->isSimple() && LSNode->isUnindexed() &&
- LSNode->getValueType(0).isScalableVector() &&
- N->getValueType(0).isSimple() && N->getValueType(0) == MemVT) {
- MVT VT = N->getValueType(0).getSimpleVT();
-
- unsigned IntID;
- switch (VT.SimpleTy) {
- default:
+ if (auto *Load = dyn_cast<LoadSDNode>(N))
+ if (tryLowerMultiVectorLoad(Load, Results, DAG))
return;
- case MVT::nxv32i8:
- case MVT::nxv16i16:
- case MVT::nxv8i32:
- case MVT::nxv4i64:
- case MVT::nxv16f16:
- case MVT::nxv8f32:
- case MVT::nxv4f64:
- case MVT::nxv16bf16:
- IntID = Intrinsic::aarch64_sve_ld1_pn_x2;
- break;
- case MVT::nxv64i8:
- case MVT::nxv32i16:
- case MVT::nxv16i32:
- case MVT::nxv8i64:
- case MVT::nxv32f16:
- case MVT::nxv16f32:
- case MVT::nxv8f64:
- case MVT::nxv32bf16:
- IntID = Intrinsic::aarch64_sve_ld1_pn_x4;
- break;
- }
-
- unsigned PredIntID;
- switch (VT.getScalarSizeInBits()) {
- default:
- llvm_unreachable("covered by previous switch");
- case 8:
- PredIntID = Intrinsic::aarch64_sve_ptrue_c8;
- break;
- case 16:
- PredIntID = Intrinsic::aarch64_sve_ptrue_c16;
- break;
- case 32:
- PredIntID = Intrinsic::aarch64_sve_ptrue_c32;
- break;
- case 64:
- PredIntID = Intrinsic::aarch64_sve_ptrue_c64;
- break;
- }
-
- SDValue Chain = LSNode->getChain();
- SDValue Addr = LSNode->getBasePtr();
-
- if (!LSNode->getOffset().isUndef())
- return;
-
- SDLoc DL(N);
- SDValue PNg =
- DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
- DAG.getConstant(PredIntID, DL, MVT::i64));
-
- if (IntID == Intrinsic::aarch64_sve_ld1_pn_x2) {
- MVT RegVT = VT.getHalfNumVectorElementsVT();
- SDValue NewLoad = DAG.getNode(
- ISD::INTRINSIC_W_CHAIN, DL, {RegVT, RegVT, MVT::Other},
- {Chain, DAG.getConstant(IntID, DL, MVT::i64), PNg, Addr});
- Results.push_back(
- DAG.getNode(ISD::CONCAT_VECTORS, DL, VT,
- {NewLoad.getValue(0), NewLoad.getValue(1)}));
- Results.push_back(NewLoad.getValue(2) /* Chain */);
- return;
- }
-
- assert(IntID == Intrinsic::aarch64_sve_ld1_pn_x4);
- MVT RegVT = VT.getHalfNumVectorElementsVT().getHalfNumVectorElementsVT();
- SDValue NewLoad = DAG.getNode(
- ISD::INTRINSIC_W_CHAIN, DL, {RegVT, RegVT, RegVT, RegVT, MVT::Other},
- {Chain, DAG.getConstant(IntID, DL, MVT::i64), PNg, Addr});
- Results.push_back(
- DAG.getNode(ISD::CONCAT_VECTORS, DL, VT,
- {NewLoad.getValue(0), NewLoad.getValue(1),
- NewLoad.getValue(2), NewLoad.getValue(3)}));
- Results.push_back(NewLoad.getValue(4) /* Chain */);
- return;
- }
if ((!LoadNode->isVolatile() && !LoadNode->isAtomic()) ||
LoadNode->getMemoryVT() != MVT::i128) {
>From af835801f0c6ec48403d1311ce71953dc562f23c Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 7 Jul 2026 16:33:03 +0000
Subject: [PATCH 4/7] rename tests
---
...oads.ll => sve-multivector-load-stores.ll} | 256 +++++++++---------
1 file changed, 128 insertions(+), 128 deletions(-)
rename llvm/test/CodeGen/AArch64/{sve-multivector-loads.ll => sve-multivector-load-stores.ll} (88%)
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll b/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
similarity index 88%
rename from llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
rename to llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
index e7a4ee355f58f..62c2f3d048844 100644
--- a/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
@@ -5,8 +5,8 @@
target triple = "aarch64-unknown-linux-gnu"
-define void @load_2x_vectors_i8_r(ptr %addr) {
-; SVE2p1-LABEL: load_2x_vectors_i8_r:
+define void @load_store_2x_vectors_i8_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_2x_vectors_i8_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #1, mul vl]
; SVE2p1-NEXT: ldr z1, [x0]
@@ -16,7 +16,7 @@ define void @load_2x_vectors_i8_r(ptr %addr) {
; SVE2p1-NEXT: str z1, [x0]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_i8_r:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_i8_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.b
; SVE2p1-SL-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0]
@@ -25,7 +25,7 @@ define void @load_2x_vectors_i8_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1b { z0.b, z1.b }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_i8_r:
+; SME2-LABEL: load_store_2x_vectors_i8_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x0]
@@ -41,8 +41,8 @@ define void @load_2x_vectors_i8_r(ptr %addr) {
ret void
}
-define void @load_2x_vectors_i8_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_2x_vectors_i8_rr:
+define void @load_store_2x_vectors_i8_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_2x_vectors_i8_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.b
; SVE2p1-NEXT: add x8, x0, x1
@@ -54,7 +54,7 @@ define void @load_2x_vectors_i8_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z1, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_i8_rr:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_i8_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.b
; SVE2p1-SL-NEXT: add x8, x0, x1
@@ -64,7 +64,7 @@ define void @load_2x_vectors_i8_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1b { z0.b, z1.b }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_i8_rr:
+; SME2-LABEL: load_store_2x_vectors_i8_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: add x8, x0, x1
@@ -82,8 +82,8 @@ define void @load_2x_vectors_i8_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_2x_vectors_i16_r(ptr %addr) {
-; SVE2p1-LABEL: load_2x_vectors_i16_r:
+define void @load_store_2x_vectors_i16_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_2x_vectors_i16_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #1, mul vl]
; SVE2p1-NEXT: ldr z1, [x0]
@@ -93,7 +93,7 @@ define void @load_2x_vectors_i16_r(ptr %addr) {
; SVE2p1-NEXT: str z1, [x0]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_i16_r:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_i16_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0]
@@ -102,7 +102,7 @@ define void @load_2x_vectors_i16_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_i16_r:
+; SME2-LABEL: load_store_2x_vectors_i16_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
@@ -118,8 +118,8 @@ define void @load_2x_vectors_i16_r(ptr %addr) {
ret void
}
-define void @load_2x_vectors_i16_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_2x_vectors_i16_rr:
+define void @load_store_2x_vectors_i16_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_2x_vectors_i16_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.h
; SVE2p1-NEXT: add x8, x0, x1, lsl #1
@@ -131,7 +131,7 @@ define void @load_2x_vectors_i16_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z1, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_i16_rr:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_i16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #1
@@ -141,7 +141,7 @@ define void @load_2x_vectors_i16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_i16_rr:
+; SME2-LABEL: load_store_2x_vectors_i16_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: add x8, x0, x1, lsl #1
@@ -159,8 +159,8 @@ define void @load_2x_vectors_i16_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_2x_vectors_i32_r(ptr %addr) {
-; SVE2p1-LABEL: load_2x_vectors_i32_r:
+define void @load_store_2x_vectors_i32_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_2x_vectors_i32_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #1, mul vl]
; SVE2p1-NEXT: ldr z1, [x0]
@@ -170,7 +170,7 @@ define void @load_2x_vectors_i32_r(ptr %addr) {
; SVE2p1-NEXT: str z1, [x0]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_i32_r:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_i32_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
; SVE2p1-SL-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0]
@@ -179,7 +179,7 @@ define void @load_2x_vectors_i32_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_i32_r:
+; SME2-LABEL: load_store_2x_vectors_i32_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0]
@@ -195,8 +195,8 @@ define void @load_2x_vectors_i32_r(ptr %addr) {
ret void
}
-define void @load_2x_vectors_i32_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_2x_vectors_i32_rr:
+define void @load_store_2x_vectors_i32_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_2x_vectors_i32_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.s
; SVE2p1-NEXT: add x8, x0, x1, lsl #2
@@ -208,7 +208,7 @@ define void @load_2x_vectors_i32_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z1, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_i32_rr:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_i32_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #2
@@ -218,7 +218,7 @@ define void @load_2x_vectors_i32_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_i32_rr:
+; SME2-LABEL: load_store_2x_vectors_i32_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: add x8, x0, x1, lsl #2
@@ -236,8 +236,8 @@ define void @load_2x_vectors_i32_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_2x_vectors_i64_r(ptr %addr) {
-; SVE2p1-LABEL: load_2x_vectors_i64_r:
+define void @load_store_2x_vectors_i64_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_2x_vectors_i64_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #1, mul vl]
; SVE2p1-NEXT: ldr z1, [x0]
@@ -247,7 +247,7 @@ define void @load_2x_vectors_i64_r(ptr %addr) {
; SVE2p1-NEXT: str z1, [x0]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_i64_r:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_i64_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
; SVE2p1-SL-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0]
@@ -256,7 +256,7 @@ define void @load_2x_vectors_i64_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_i64_r:
+; SME2-LABEL: load_store_2x_vectors_i64_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0]
@@ -272,8 +272,8 @@ define void @load_2x_vectors_i64_r(ptr %addr) {
ret void
}
-define void @load_2x_vectors_i64_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_2x_vectors_i64_rr:
+define void @load_store_2x_vectors_i64_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_2x_vectors_i64_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.d
; SVE2p1-NEXT: add x8, x0, x1, lsl #3
@@ -285,7 +285,7 @@ define void @load_2x_vectors_i64_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z1, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_i64_rr:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_i64_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #3
@@ -295,7 +295,7 @@ define void @load_2x_vectors_i64_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_i64_rr:
+; SME2-LABEL: load_store_2x_vectors_i64_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: add x8, x0, x1, lsl #3
@@ -313,8 +313,8 @@ define void @load_2x_vectors_i64_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_2x_vectors_f16_r(ptr %addr) {
-; SVE2p1-LABEL: load_2x_vectors_f16_r:
+define void @load_store_2x_vectors_f16_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_2x_vectors_f16_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #1, mul vl]
; SVE2p1-NEXT: ldr z1, [x0]
@@ -325,7 +325,7 @@ define void @load_2x_vectors_f16_r(ptr %addr) {
; SVE2p1-NEXT: str z1, [x0]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_f16_r:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_f16_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: ptrue p0.h
@@ -335,7 +335,7 @@ define void @load_2x_vectors_f16_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_f16_r:
+; SME2-LABEL: load_store_2x_vectors_f16_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ptrue p0.h
@@ -352,8 +352,8 @@ define void @load_2x_vectors_f16_r(ptr %addr) {
ret void
}
-define void @load_2x_vectors_f16_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_2x_vectors_f16_rr:
+define void @load_store_2x_vectors_f16_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_2x_vectors_f16_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.h
; SVE2p1-NEXT: add x8, x0, x1, lsl #1
@@ -365,7 +365,7 @@ define void @load_2x_vectors_f16_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z1, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_f16_rr:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_f16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: ptrue p0.h
@@ -376,7 +376,7 @@ define void @load_2x_vectors_f16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_f16_rr:
+; SME2-LABEL: load_store_2x_vectors_f16_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ptrue p0.h
@@ -395,8 +395,8 @@ define void @load_2x_vectors_f16_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_2x_vectors_f32_r(ptr %addr) {
-; SVE2p1-LABEL: load_2x_vectors_f32_r:
+define void @load_store_2x_vectors_f32_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_2x_vectors_f32_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #1, mul vl]
; SVE2p1-NEXT: ldr z1, [x0]
@@ -407,7 +407,7 @@ define void @load_2x_vectors_f32_r(ptr %addr) {
; SVE2p1-NEXT: str z1, [x0]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_f32_r:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_f32_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
; SVE2p1-SL-NEXT: ptrue p0.s
@@ -417,7 +417,7 @@ define void @load_2x_vectors_f32_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_f32_r:
+; SME2-LABEL: load_store_2x_vectors_f32_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ptrue p0.s
@@ -434,8 +434,8 @@ define void @load_2x_vectors_f32_r(ptr %addr) {
ret void
}
-define void @load_2x_vectors_f32_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_2x_vectors_f32_rr:
+define void @load_store_2x_vectors_f32_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_2x_vectors_f32_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.s
; SVE2p1-NEXT: add x8, x0, x1, lsl #2
@@ -447,7 +447,7 @@ define void @load_2x_vectors_f32_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z1, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_f32_rr:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_f32_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
; SVE2p1-SL-NEXT: ptrue p0.s
@@ -458,7 +458,7 @@ define void @load_2x_vectors_f32_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_f32_rr:
+; SME2-LABEL: load_store_2x_vectors_f32_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ptrue p0.s
@@ -477,8 +477,8 @@ define void @load_2x_vectors_f32_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_2x_vectors_f64_r(ptr %addr) {
-; SVE2p1-LABEL: load_2x_vectors_f64_r:
+define void @load_store_2x_vectors_f64_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_2x_vectors_f64_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #1, mul vl]
; SVE2p1-NEXT: ldr z1, [x0]
@@ -489,7 +489,7 @@ define void @load_2x_vectors_f64_r(ptr %addr) {
; SVE2p1-NEXT: str z1, [x0]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_f64_r:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_f64_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
; SVE2p1-SL-NEXT: ptrue p0.d
@@ -499,7 +499,7 @@ define void @load_2x_vectors_f64_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_f64_r:
+; SME2-LABEL: load_store_2x_vectors_f64_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ptrue p0.d
@@ -516,8 +516,8 @@ define void @load_2x_vectors_f64_r(ptr %addr) {
ret void
}
-define void @load_2x_vectors_f64_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_2x_vectors_f64_rr:
+define void @load_store_2x_vectors_f64_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_2x_vectors_f64_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.d
; SVE2p1-NEXT: add x8, x0, x1, lsl #3
@@ -529,7 +529,7 @@ define void @load_2x_vectors_f64_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z1, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_f64_rr:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_f64_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
; SVE2p1-SL-NEXT: ptrue p0.d
@@ -540,7 +540,7 @@ define void @load_2x_vectors_f64_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_f64_rr:
+; SME2-LABEL: load_store_2x_vectors_f64_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ptrue p0.d
@@ -559,8 +559,8 @@ define void @load_2x_vectors_f64_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_2x_vectors_bf16_r(ptr %addr) {
-; SVE2p1-LABEL: load_2x_vectors_bf16_r:
+define void @load_store_2x_vectors_bf16_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_2x_vectors_bf16_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: fmov z0.h, #1.87500000
; SVE2p1-NEXT: ldr z1, [x0, #1, mul vl]
@@ -571,7 +571,7 @@ define void @load_2x_vectors_bf16_r(ptr %addr) {
; SVE2p1-NEXT: str z0, [x0]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_bf16_r:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_bf16_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: fmov z2.h, #1.87500000
@@ -581,7 +581,7 @@ define void @load_2x_vectors_bf16_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_bf16_r:
+; SME2-LABEL: load_store_2x_vectors_bf16_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: fmov z0.h, #1.87500000
@@ -596,8 +596,8 @@ define void @load_2x_vectors_bf16_r(ptr %addr) {
ret void
}
-define void @load_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_2x_vectors_bf16_rr:
+define void @load_store_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_2x_vectors_bf16_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.h
; SVE2p1-NEXT: add x8, x0, x1, lsl #1
@@ -610,7 +610,7 @@ define void @load_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z1, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_2x_vectors_bf16_rr:
+; SVE2p1-SL-LABEL: load_store_2x_vectors_bf16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: fmov z2.h, #1.87500000
@@ -621,7 +621,7 @@ define void @load_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_2x_vectors_bf16_rr:
+; SME2-LABEL: load_store_2x_vectors_bf16_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: fmov z0.h, #1.87500000
@@ -640,8 +640,8 @@ define void @load_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
-define void @load_4x_vectors_i8_r(ptr %addr) {
-; SVE2p1-LABEL: load_4x_vectors_i8_r:
+define void @load_store_4x_vectors_i8_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_4x_vectors_i8_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
@@ -657,7 +657,7 @@ define void @load_4x_vectors_i8_r(ptr %addr) {
; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_i8_r:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_i8_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.b
; SVE2p1-SL-NEXT: ld1b { z0.b - z3.b }, pn8/z, [x0]
@@ -668,7 +668,7 @@ define void @load_4x_vectors_i8_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1b { z0.b - z3.b }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_i8_r:
+; SME2-LABEL: load_store_4x_vectors_i8_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
@@ -688,8 +688,8 @@ define void @load_4x_vectors_i8_r(ptr %addr) {
ret void
}
-define void @load_4x_vectors_i8_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_4x_vectors_i8_rr:
+define void @load_store_4x_vectors_i8_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_4x_vectors_i8_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.b
; SVE2p1-NEXT: add x8, x0, x1
@@ -707,7 +707,7 @@ define void @load_4x_vectors_i8_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_i8_rr:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_i8_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.b
; SVE2p1-SL-NEXT: add x8, x0, x1
@@ -719,7 +719,7 @@ define void @load_4x_vectors_i8_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1b { z0.b - z3.b }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_i8_rr:
+; SME2-LABEL: load_store_4x_vectors_i8_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: add x8, x0, x1
@@ -741,8 +741,8 @@ define void @load_4x_vectors_i8_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_4x_vectors_i16_r(ptr %addr) {
-; SVE2p1-LABEL: load_4x_vectors_i16_r:
+define void @load_store_4x_vectors_i16_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_4x_vectors_i16_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
@@ -758,7 +758,7 @@ define void @load_4x_vectors_i16_r(ptr %addr) {
; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_i16_r:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_i16_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: ld1h { z0.h - z3.h }, pn8/z, [x0]
@@ -769,7 +769,7 @@ define void @load_4x_vectors_i16_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_i16_r:
+; SME2-LABEL: load_store_4x_vectors_i16_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
@@ -789,8 +789,8 @@ define void @load_4x_vectors_i16_r(ptr %addr) {
ret void
}
-define void @load_4x_vectors_i16_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_4x_vectors_i16_rr:
+define void @load_store_4x_vectors_i16_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_4x_vectors_i16_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.h
; SVE2p1-NEXT: add x8, x0, x1, lsl #1
@@ -808,7 +808,7 @@ define void @load_4x_vectors_i16_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_i16_rr:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_i16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #1
@@ -820,7 +820,7 @@ define void @load_4x_vectors_i16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_i16_rr:
+; SME2-LABEL: load_store_4x_vectors_i16_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: add x8, x0, x1, lsl #1
@@ -842,8 +842,8 @@ define void @load_4x_vectors_i16_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_4x_vectors_i32_r(ptr %addr) {
-; SVE2p1-LABEL: load_4x_vectors_i32_r:
+define void @load_store_4x_vectors_i32_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_4x_vectors_i32_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
@@ -859,7 +859,7 @@ define void @load_4x_vectors_i32_r(ptr %addr) {
; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_i32_r:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_i32_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
; SVE2p1-SL-NEXT: ld1w { z0.s - z3.s }, pn8/z, [x0]
@@ -870,7 +870,7 @@ define void @load_4x_vectors_i32_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1w { z0.s - z3.s }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_i32_r:
+; SME2-LABEL: load_store_4x_vectors_i32_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0]
@@ -890,8 +890,8 @@ define void @load_4x_vectors_i32_r(ptr %addr) {
ret void
}
-define void @load_4x_vectors_i32_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_4x_vectors_i32_rr:
+define void @load_store_4x_vectors_i32_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_4x_vectors_i32_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.s
; SVE2p1-NEXT: add x8, x0, x1, lsl #2
@@ -909,7 +909,7 @@ define void @load_4x_vectors_i32_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_i32_rr:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_i32_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #2
@@ -921,7 +921,7 @@ define void @load_4x_vectors_i32_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1w { z0.s - z3.s }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_i32_rr:
+; SME2-LABEL: load_store_4x_vectors_i32_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: add x8, x0, x1, lsl #2
@@ -943,8 +943,8 @@ define void @load_4x_vectors_i32_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_4x_vectors_i64_r(ptr %addr) {
-; SVE2p1-LABEL: load_4x_vectors_i64_r:
+define void @load_store_4x_vectors_i64_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_4x_vectors_i64_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
@@ -960,7 +960,7 @@ define void @load_4x_vectors_i64_r(ptr %addr) {
; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_i64_r:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_i64_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
; SVE2p1-SL-NEXT: ld1d { z0.d - z3.d }, pn8/z, [x0]
@@ -971,7 +971,7 @@ define void @load_4x_vectors_i64_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1d { z0.d - z3.d }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_i64_r:
+; SME2-LABEL: load_store_4x_vectors_i64_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0]
@@ -991,8 +991,8 @@ define void @load_4x_vectors_i64_r(ptr %addr) {
ret void
}
-define void @load_4x_vectors_i64_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_4x_vectors_i64_rr:
+define void @load_store_4x_vectors_i64_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_4x_vectors_i64_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.d
; SVE2p1-NEXT: add x8, x0, x1, lsl #3
@@ -1010,7 +1010,7 @@ define void @load_4x_vectors_i64_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_i64_rr:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_i64_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #3
@@ -1022,7 +1022,7 @@ define void @load_4x_vectors_i64_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1d { z0.d - z3.d }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_i64_rr:
+; SME2-LABEL: load_store_4x_vectors_i64_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: add x8, x0, x1, lsl #3
@@ -1044,8 +1044,8 @@ define void @load_4x_vectors_i64_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_4x_vectors_f16_r(ptr %addr) {
-; SVE2p1-LABEL: load_4x_vectors_f16_r:
+define void @load_store_4x_vectors_f16_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_4x_vectors_f16_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
@@ -1062,7 +1062,7 @@ define void @load_4x_vectors_f16_r(ptr %addr) {
; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_f16_r:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_f16_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: ptrue p0.h
@@ -1074,7 +1074,7 @@ define void @load_4x_vectors_f16_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_f16_r:
+; SME2-LABEL: load_store_4x_vectors_f16_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ptrue p0.h
@@ -1095,8 +1095,8 @@ define void @load_4x_vectors_f16_r(ptr %addr) {
ret void
}
-define void @load_4x_vectors_f16_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_4x_vectors_f16_rr:
+define void @load_store_4x_vectors_f16_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_4x_vectors_f16_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.h
; SVE2p1-NEXT: add x8, x0, x1, lsl #1
@@ -1114,7 +1114,7 @@ define void @load_4x_vectors_f16_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_f16_rr:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_f16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: ptrue p0.h
@@ -1127,7 +1127,7 @@ define void @load_4x_vectors_f16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_f16_rr:
+; SME2-LABEL: load_store_4x_vectors_f16_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ptrue p0.h
@@ -1150,8 +1150,8 @@ define void @load_4x_vectors_f16_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_4x_vectors_f32_r(ptr %addr) {
-; SVE2p1-LABEL: load_4x_vectors_f32_r:
+define void @load_store_4x_vectors_f32_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_4x_vectors_f32_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
@@ -1168,7 +1168,7 @@ define void @load_4x_vectors_f32_r(ptr %addr) {
; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_f32_r:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_f32_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
; SVE2p1-SL-NEXT: ptrue p0.s
@@ -1180,7 +1180,7 @@ define void @load_4x_vectors_f32_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1w { z0.s - z3.s }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_f32_r:
+; SME2-LABEL: load_store_4x_vectors_f32_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ptrue p0.s
@@ -1201,8 +1201,8 @@ define void @load_4x_vectors_f32_r(ptr %addr) {
ret void
}
-define void @load_4x_vectors_f32_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_4x_vectors_f32_rr:
+define void @load_store_4x_vectors_f32_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_4x_vectors_f32_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.s
; SVE2p1-NEXT: add x8, x0, x1, lsl #2
@@ -1220,7 +1220,7 @@ define void @load_4x_vectors_f32_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_f32_rr:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_f32_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
; SVE2p1-SL-NEXT: ptrue p0.s
@@ -1233,7 +1233,7 @@ define void @load_4x_vectors_f32_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1w { z0.s - z3.s }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_f32_rr:
+; SME2-LABEL: load_store_4x_vectors_f32_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ptrue p0.s
@@ -1256,8 +1256,8 @@ define void @load_4x_vectors_f32_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_4x_vectors_f64_r(ptr %addr) {
-; SVE2p1-LABEL: load_4x_vectors_f64_r:
+define void @load_store_4x_vectors_f64_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_4x_vectors_f64_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ldr z0, [x0, #2, mul vl]
; SVE2p1-NEXT: ldr z1, [x0, #3, mul vl]
@@ -1274,7 +1274,7 @@ define void @load_4x_vectors_f64_r(ptr %addr) {
; SVE2p1-NEXT: str z3, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_f64_r:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_f64_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
; SVE2p1-SL-NEXT: ptrue p0.d
@@ -1286,7 +1286,7 @@ define void @load_4x_vectors_f64_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1d { z0.d - z3.d }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_f64_r:
+; SME2-LABEL: load_store_4x_vectors_f64_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ptrue p0.d
@@ -1307,8 +1307,8 @@ define void @load_4x_vectors_f64_r(ptr %addr) {
ret void
}
-define void @load_4x_vectors_f64_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_4x_vectors_f64_rr:
+define void @load_store_4x_vectors_f64_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_4x_vectors_f64_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.d
; SVE2p1-NEXT: add x8, x0, x1, lsl #3
@@ -1326,7 +1326,7 @@ define void @load_4x_vectors_f64_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z3, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_f64_rr:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_f64_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
; SVE2p1-SL-NEXT: ptrue p0.d
@@ -1339,7 +1339,7 @@ define void @load_4x_vectors_f64_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1d { z0.d - z3.d }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_f64_rr:
+; SME2-LABEL: load_store_4x_vectors_f64_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ptrue p0.d
@@ -1362,8 +1362,8 @@ define void @load_4x_vectors_f64_rr(ptr %base, i64 %idx) {
ret void
}
-define void @load_4x_vectors_bf16_r(ptr %addr) {
-; SVE2p1-LABEL: load_4x_vectors_bf16_r:
+define void @load_store_4x_vectors_bf16_r(ptr %addr) {
+; SVE2p1-LABEL: load_store_4x_vectors_bf16_r:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: fmov z0.h, #1.87500000
; SVE2p1-NEXT: ldr z1, [x0, #2, mul vl]
@@ -1380,7 +1380,7 @@ define void @load_4x_vectors_bf16_r(ptr %addr) {
; SVE2p1-NEXT: str z0, [x0, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_bf16_r:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_bf16_r:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: fmov z4.h, #1.87500000
@@ -1392,7 +1392,7 @@ define void @load_4x_vectors_bf16_r(ptr %addr) {
; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_bf16_r:
+; SME2-LABEL: load_store_4x_vectors_bf16_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: fmov z0.h, #1.87500000
@@ -1409,8 +1409,8 @@ define void @load_4x_vectors_bf16_r(ptr %addr) {
ret void
}
-define void @load_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
-; SVE2p1-LABEL: load_4x_vectors_bf16_rr:
+define void @load_store_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
+; SVE2p1-LABEL: load_store_4x_vectors_bf16_rr:
; SVE2p1: // %bb.0:
; SVE2p1-NEXT: ptrue p0.h
; SVE2p1-NEXT: add x8, x0, x1, lsl #1
@@ -1429,7 +1429,7 @@ define void @load_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SVE2p1-NEXT: str z0, [x8, #1, mul vl]
; SVE2p1-NEXT: ret
;
-; SVE2p1-SL-LABEL: load_4x_vectors_bf16_rr:
+; SVE2p1-SL-LABEL: load_store_4x_vectors_bf16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
; SVE2p1-SL-NEXT: fmov z4.h, #1.87500000
@@ -1442,7 +1442,7 @@ define void @load_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: st1h { z0.h - z3.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
-; SME2-LABEL: load_4x_vectors_bf16_rr:
+; SME2-LABEL: load_store_4x_vectors_bf16_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: fmov z0.h, #1.87500000
>From 15c35a3cb630d1dcfc0cf7d5f206cc017221b42b Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 10 Jul 2026 10:25:12 +0000
Subject: [PATCH 5/7] Return chain if value is undef
---
.../Target/AArch64/AArch64ISelLowering.cpp | 3 +
.../CodeGen/AArch64/sve-vector-interleave.ll | 204 ++++++------------
2 files changed, 63 insertions(+), 144 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 883fce00b6aa4..d059c880fa9c7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -7815,6 +7815,9 @@ static SDValue tryLowerMultiVectorStore(StoreSDNode *StoreNode,
if (!isValidSVEMultiVectorOp(StoreNode, VT))
return SDValue();
+ if (Value->isUndef())
+ return StoreNode->getChain();
+
MVT StoreVT = VT.getSimpleVT();
std::optional<SVEMultiVectorInfo> MultiVecInfo =
getSVEMultiVectorInfo(StoreVT);
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
index d5de1761e9a49..81a0cc242fc11 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
@@ -204,54 +204,26 @@ define <vscale x 6 x half> @interleave3_nxv6f16(<vscale x 2 x half> %vec0, <vsca
}
define <vscale x 12 x half> @interleave3_nxv12f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1, <vscale x 4 x half> %vec2) {
-; SVE-LABEL: interleave3_nxv12f16:
-; SVE: // %bb.0:
-; SVE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SVE-NEXT: addvl sp, sp, #-5
-; SVE-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
-; SVE-NEXT: .cfi_offset w29, -16
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: addpl x8, sp, #4
-; SVE-NEXT: st3w { z0.s - z2.s }, p0, [sp]
-; SVE-NEXT: ldr z0, [sp, #1, mul vl]
-; SVE-NEXT: ldr z1, [sp]
-; SVE-NEXT: ldr z2, [sp, #2, mul vl]
-; SVE-NEXT: uzp1 z0.h, z1.h, z0.h
-; SVE-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
-; SVE-NEXT: str z0, [sp, #3, mul vl]
-; SVE-NEXT: ldr z1, [sp, #4, mul vl]
-; SVE-NEXT: ldr z0, [sp, #3, mul vl]
-; SVE-NEXT: addvl sp, sp, #5
-; SVE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; SVE-NEXT: ret
-;
-; SME2-LABEL: interleave3_nxv12f16:
-; SME2: // %bb.0:
-; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
-; SME2-NEXT: addvl sp, sp, #-5
-; SME2-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x30, 0x1e, 0x22 // sp + 16 + 48 * VG
-; SME2-NEXT: .cfi_offset w29, -16
-; SME2-NEXT: ptrue p0.s
-; SME2-NEXT: ptrue pn8.h
-; SME2-NEXT: addvl x8, sp, #1
-; SME2-NEXT: st3w { z0.s - z2.s }, p0, [sp]
-; SME2-NEXT: ldr z0, [sp, #1, mul vl]
-; SME2-NEXT: ldr z1, [sp]
-; SME2-NEXT: ldr z2, [sp, #2, mul vl]
-; SME2-NEXT: uzp1 z0.h, z1.h, z0.h
-; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8, #2, mul vl]
-; SME2-NEXT: addpl x8, sp, #4
-; SME2-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
-; SME2-NEXT: str z0, [sp, #3, mul vl]
-; SME2-NEXT: ldr z1, [sp, #4, mul vl]
-; SME2-NEXT: ldr z0, [sp, #3, mul vl]
-; SME2-NEXT: addvl sp, sp, #5
-; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
-; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; SME2-NEXT: ret
+; CHECK-LABEL: interleave3_nxv12f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-5
+; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: addpl x8, sp, #4
+; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [sp]
+; CHECK-NEXT: ldr z0, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z1, [sp]
+; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
+; CHECK-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
+; CHECK-NEXT: str z0, [sp, #3, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #4, mul vl]
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl]
+; CHECK-NEXT: addvl sp, sp, #5
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
%retval = call <vscale x 12 x half> @llvm.vector.interleave3.nxv12f16(<vscale x 4 x half> %vec0, <vscale x 4 x half> %vec1, <vscale x 4 x half> %vec2)
ret <vscale x 12 x half> %retval
}
@@ -276,54 +248,26 @@ define <vscale x 24 x half> @interleave3_nxv24f16(<vscale x 8 x half> %vec0, <vs
}
define <vscale x 6 x float> @interleave3_nxv6f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1, <vscale x 2 x float> %vec2) {
-; SVE-LABEL: interleave3_nxv6f32:
-; SVE: // %bb.0:
-; SVE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SVE-NEXT: addvl sp, sp, #-5
-; SVE-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
-; SVE-NEXT: .cfi_offset w29, -16
-; SVE-NEXT: ptrue p0.d
-; SVE-NEXT: addpl x8, sp, #4
-; SVE-NEXT: st3d { z0.d - z2.d }, p0, [sp]
-; SVE-NEXT: ldr z0, [sp, #1, mul vl]
-; SVE-NEXT: ldr z1, [sp]
-; SVE-NEXT: ldr z2, [sp, #2, mul vl]
-; SVE-NEXT: uzp1 z0.s, z1.s, z0.s
-; SVE-NEXT: st1w { z2.d }, p0, [x8, #7, mul vl]
-; SVE-NEXT: str z0, [sp, #3, mul vl]
-; SVE-NEXT: ldr z1, [sp, #4, mul vl]
-; SVE-NEXT: ldr z0, [sp, #3, mul vl]
-; SVE-NEXT: addvl sp, sp, #5
-; SVE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; SVE-NEXT: ret
-;
-; SME2-LABEL: interleave3_nxv6f32:
-; SME2: // %bb.0:
-; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
-; SME2-NEXT: addvl sp, sp, #-5
-; SME2-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x30, 0x1e, 0x22 // sp + 16 + 48 * VG
-; SME2-NEXT: .cfi_offset w29, -16
-; SME2-NEXT: ptrue p0.d
-; SME2-NEXT: ptrue pn8.s
-; SME2-NEXT: addvl x8, sp, #1
-; SME2-NEXT: st3d { z0.d - z2.d }, p0, [sp]
-; SME2-NEXT: ldr z0, [sp, #1, mul vl]
-; SME2-NEXT: ldr z1, [sp]
-; SME2-NEXT: ldr z2, [sp, #2, mul vl]
-; SME2-NEXT: uzp1 z0.s, z1.s, z0.s
-; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x8, #2, mul vl]
-; SME2-NEXT: addpl x8, sp, #4
-; SME2-NEXT: st1w { z2.d }, p0, [x8, #7, mul vl]
-; SME2-NEXT: str z0, [sp, #3, mul vl]
-; SME2-NEXT: ldr z1, [sp, #4, mul vl]
-; SME2-NEXT: ldr z0, [sp, #3, mul vl]
-; SME2-NEXT: addvl sp, sp, #5
-; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
-; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; SME2-NEXT: ret
+; CHECK-LABEL: interleave3_nxv6f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-5
+; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: addpl x8, sp, #4
+; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [sp]
+; CHECK-NEXT: ldr z0, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z1, [sp]
+; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: uzp1 z0.s, z1.s, z0.s
+; CHECK-NEXT: st1w { z2.d }, p0, [x8, #7, mul vl]
+; CHECK-NEXT: str z0, [sp, #3, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #4, mul vl]
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl]
+; CHECK-NEXT: addvl sp, sp, #5
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
%retval = call <vscale x 6 x float> @llvm.vector.interleave3.nxv6f32(<vscale x 2 x float> %vec0, <vscale x 2 x float> %vec1, <vscale x 2 x float> %vec2)
ret <vscale x 6 x float> %retval
}
@@ -389,54 +333,26 @@ define <vscale x 6 x bfloat> @interleave3_nxv6bf16(<vscale x 2 x bfloat> %vec0,
}
define <vscale x 12 x bfloat> @interleave3_nxv12bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1, <vscale x 4 x bfloat> %vec2) {
-; SVE-LABEL: interleave3_nxv12bf16:
-; SVE: // %bb.0:
-; SVE-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SVE-NEXT: addvl sp, sp, #-5
-; SVE-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
-; SVE-NEXT: .cfi_offset w29, -16
-; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: addpl x8, sp, #4
-; SVE-NEXT: st3w { z0.s - z2.s }, p0, [sp]
-; SVE-NEXT: ldr z0, [sp, #1, mul vl]
-; SVE-NEXT: ldr z1, [sp]
-; SVE-NEXT: ldr z2, [sp, #2, mul vl]
-; SVE-NEXT: uzp1 z0.h, z1.h, z0.h
-; SVE-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
-; SVE-NEXT: str z0, [sp, #3, mul vl]
-; SVE-NEXT: ldr z1, [sp, #4, mul vl]
-; SVE-NEXT: ldr z0, [sp, #3, mul vl]
-; SVE-NEXT: addvl sp, sp, #5
-; SVE-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; SVE-NEXT: ret
-;
-; SME2-LABEL: interleave3_nxv12bf16:
-; SME2: // %bb.0:
-; SME2-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
-; SME2-NEXT: addvl sp, sp, #-1
-; SME2-NEXT: str p8, [sp, #7, mul vl] // 2-byte Spill
-; SME2-NEXT: addvl sp, sp, #-5
-; SME2-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x30, 0x1e, 0x22 // sp + 16 + 48 * VG
-; SME2-NEXT: .cfi_offset w29, -16
-; SME2-NEXT: ptrue p0.s
-; SME2-NEXT: ptrue pn8.h
-; SME2-NEXT: addvl x8, sp, #1
-; SME2-NEXT: st3w { z0.s - z2.s }, p0, [sp]
-; SME2-NEXT: ldr z0, [sp, #1, mul vl]
-; SME2-NEXT: ldr z1, [sp]
-; SME2-NEXT: ldr z2, [sp, #2, mul vl]
-; SME2-NEXT: uzp1 z0.h, z1.h, z0.h
-; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8, #2, mul vl]
-; SME2-NEXT: addpl x8, sp, #4
-; SME2-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
-; SME2-NEXT: str z0, [sp, #3, mul vl]
-; SME2-NEXT: ldr z1, [sp, #4, mul vl]
-; SME2-NEXT: ldr z0, [sp, #3, mul vl]
-; SME2-NEXT: addvl sp, sp, #5
-; SME2-NEXT: ldr p8, [sp, #7, mul vl] // 2-byte Reload
-; SME2-NEXT: addvl sp, sp, #1
-; SME2-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
-; SME2-NEXT: ret
+; CHECK-LABEL: interleave3_nxv12bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-5
+; CHECK-NEXT: .cfi_escape 0x0f, 0x09, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x11, 0x28, 0x1e, 0x22 // sp + 16 + 40 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: addpl x8, sp, #4
+; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [sp]
+; CHECK-NEXT: ldr z0, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z1, [sp]
+; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: uzp1 z0.h, z1.h, z0.h
+; CHECK-NEXT: st1h { z2.s }, p0, [x8, #7, mul vl]
+; CHECK-NEXT: str z0, [sp, #3, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #4, mul vl]
+; CHECK-NEXT: ldr z0, [sp, #3, mul vl]
+; CHECK-NEXT: addvl sp, sp, #5
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
%retval = call <vscale x 12 x bfloat> @llvm.vector.interleave3.nxv12bf16(<vscale x 4 x bfloat> %vec0, <vscale x 4 x bfloat> %vec1, <vscale x 4 x bfloat> %vec2)
ret <vscale x 12 x bfloat> %retval
}
>From 649bbeea2d277a41c52f42c84d45533bad05a33d Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 14 Jul 2026 16:01:46 +0000
Subject: [PATCH 6/7] Refactor MultiVectorInfo
---
.../Target/AArch64/AArch64ISelLowering.cpp | 116 ++++++++----------
1 file changed, 49 insertions(+), 67 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d059c880fa9c7..d5f83362e6702 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -6209,6 +6209,30 @@ static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
DAG.getTargetConstant(Pattern, DL, MVT::i32));
}
+static inline SDValue getPTrueAsCounter(SelectionDAG &DAG, SDLoc DL, EVT VT) {
+ Intrinsic::ID IID;
+
+ switch (VT.getScalarSizeInBits()) {
+ default:
+ llvm_unreachable("unsupported predicate element size");
+ case 8:
+ IID = Intrinsic::aarch64_sve_ptrue_c8;
+ break;
+ case 16:
+ IID = Intrinsic::aarch64_sve_ptrue_c16;
+ break;
+ case 32:
+ IID = Intrinsic::aarch64_sve_ptrue_c32;
+ break;
+ case 64:
+ IID = Intrinsic::aarch64_sve_ptrue_c64;
+ break;
+ }
+
+ return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
+ DAG.getConstant(IID, DL, MVT::i64));
+}
+
static SDValue optimizeIncrementingWhile(SDNode *N, SelectionDAG &DAG,
bool IsSigned, bool IsEqual) {
unsigned Op0 = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 1 : 0;
@@ -7741,65 +7765,33 @@ struct SVEMultiVectorInfo {
unsigned NumVecs;
Intrinsic::ID LoadIntID;
Intrinsic::ID StoreIntID;
- Intrinsic::ID PTrueIntID;
};
-static std::optional<SVEMultiVectorInfo> getSVEMultiVectorInfo(MVT VT) {
+static SVEMultiVectorInfo getSVEMultiVectorInfo(MVT VT) {
SVEMultiVectorInfo Info;
- switch (VT.SimpleTy) {
- default:
- return std::nullopt;
+ TypeSize Size = VT.getSizeInBits();
+
+ assert((Size == TypeSize::getScalable(2 * 128) ||
+ Size == TypeSize::getScalable(4 * 128)) &&
+ "invalid SVE multi-vector size");
- case MVT::nxv32i8:
- case MVT::nxv16i16:
- case MVT::nxv8i32:
- case MVT::nxv4i64:
- case MVT::nxv16f16:
- case MVT::nxv8f32:
- case MVT::nxv4f64:
- case MVT::nxv16bf16:
+ Info.RegVT = getPackedSVEVectorVT(VT.getVectorElementType()).getSimpleVT();
+
+ if (Size == TypeSize::getScalable(2 * 128)) {
Info.LoadIntID = Intrinsic::aarch64_sve_ld1_pn_x2;
Info.StoreIntID = Intrinsic::aarch64_sve_st1_pn_x2;
Info.NumVecs = 2;
- Info.RegVT = VT.getHalfNumVectorElementsVT();
- break;
- case MVT::nxv64i8:
- case MVT::nxv32i16:
- case MVT::nxv16i32:
- case MVT::nxv8i64:
- case MVT::nxv32f16:
- case MVT::nxv16f32:
- case MVT::nxv8f64:
- case MVT::nxv32bf16:
+ } else {
Info.LoadIntID = Intrinsic::aarch64_sve_ld1_pn_x4;
Info.StoreIntID = Intrinsic::aarch64_sve_st1_pn_x4;
Info.NumVecs = 4;
- Info.RegVT = VT.getHalfNumVectorElementsVT().getHalfNumVectorElementsVT();
- break;
- }
-
- switch (VT.getScalarSizeInBits()) {
- default:
- llvm_unreachable("covered by previous switch");
- case 8:
- Info.PTrueIntID = Intrinsic::aarch64_sve_ptrue_c8;
- break;
- case 16:
- Info.PTrueIntID = Intrinsic::aarch64_sve_ptrue_c16;
- break;
- case 32:
- Info.PTrueIntID = Intrinsic::aarch64_sve_ptrue_c32;
- break;
- case 64:
- Info.PTrueIntID = Intrinsic::aarch64_sve_ptrue_c64;
- break;
}
return Info;
}
-static bool isValidSVEMultiVectorOp(const LSBaseSDNode *LSNode, EVT VT) {
+static bool isSimpleScalableLoadOrStore(const LSBaseSDNode *LSNode, EVT VT) {
return LSNode->isSimple() && LSNode->isUnindexed() &&
LSNode->getOffset().isUndef() && VT.isScalableVector() &&
VT.isSimple() && VT == LSNode->getMemoryVT();
@@ -7812,30 +7804,25 @@ static SDValue tryLowerMultiVectorStore(StoreSDNode *StoreNode,
SDValue Value = StoreNode->getValue();
EVT VT = Value.getValueType();
- if (!isValidSVEMultiVectorOp(StoreNode, VT))
+ if (!isSimpleScalableLoadOrStore(StoreNode, VT))
return SDValue();
if (Value->isUndef())
return StoreNode->getChain();
MVT StoreVT = VT.getSimpleVT();
- std::optional<SVEMultiVectorInfo> MultiVecInfo =
- getSVEMultiVectorInfo(StoreVT);
- if (!MultiVecInfo)
- return SDValue();
+ SVEMultiVectorInfo MultiVecInfo = getSVEMultiVectorInfo(StoreVT);
SDLoc DL(StoreNode);
- SDValue PNg =
- DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
- DAG.getConstant(MultiVecInfo->PTrueIntID, DL, MVT::i64));
+ SDValue PNg = getPTrueAsCounter(DAG, DL, VT);
SmallVector<SDValue, 8> Ops;
Ops.push_back(StoreNode->getChain());
- Ops.push_back(DAG.getConstant(MultiVecInfo->StoreIntID, DL, MVT::i64));
+ Ops.push_back(DAG.getConstant(MultiVecInfo.StoreIntID, DL, MVT::i64));
- unsigned RegElts = MultiVecInfo->RegVT.getVectorMinNumElements();
- for (unsigned i = 0; i != MultiVecInfo->NumVecs; ++i)
- Ops.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MultiVecInfo->RegVT,
+ unsigned RegElts = MultiVecInfo.RegVT.getVectorMinNumElements();
+ for (unsigned i = 0; i != MultiVecInfo.NumVecs; ++i)
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MultiVecInfo.RegVT,
Value,
DAG.getVectorIdxConstant(i * RegElts, DL)));
@@ -7852,34 +7839,29 @@ static bool tryLowerMultiVectorLoad(LoadSDNode *LoadNode,
SelectionDAG &DAG) {
EVT VT = LoadNode->getValueType(0);
- if (!isValidSVEMultiVectorOp(LoadNode, VT))
+ if (!isSimpleScalableLoadOrStore(LoadNode, VT))
return false;
MVT LoadVT = VT.getSimpleVT();
- std::optional<SVEMultiVectorInfo> MultiVecInfo =
- getSVEMultiVectorInfo(LoadVT);
- if (!MultiVecInfo)
- return false;
+ SVEMultiVectorInfo MultiVecInfo = getSVEMultiVectorInfo(LoadVT);
SDLoc DL(LoadNode);
- SDValue PNg =
- DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
- DAG.getConstant(MultiVecInfo->PTrueIntID, DL, MVT::i64));
+ SDValue PNg = getPTrueAsCounter(DAG, DL, VT);
- SmallVector<EVT, 5> ResultVTs(MultiVecInfo->NumVecs, MultiVecInfo->RegVT);
+ SmallVector<EVT, 5> ResultVTs(MultiVecInfo.NumVecs, MultiVecInfo.RegVT);
ResultVTs.push_back(MVT::Other);
SDValue NewLoad =
DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, ResultVTs,
{LoadNode->getChain(),
- DAG.getConstant(MultiVecInfo->LoadIntID, DL, MVT::i64), PNg,
+ DAG.getConstant(MultiVecInfo.LoadIntID, DL, MVT::i64), PNg,
LoadNode->getBasePtr()});
SmallVector<SDValue, 4> ResultOps;
- for (unsigned I = 0; I != MultiVecInfo->NumVecs; ++I)
+ for (unsigned I = 0; I != MultiVecInfo.NumVecs; ++I)
ResultOps.push_back(NewLoad.getValue(I));
Results.push_back(DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResultOps));
- Results.push_back(NewLoad.getValue(MultiVecInfo->NumVecs) /* Chain */);
+ Results.push_back(NewLoad.getValue(MultiVecInfo.NumVecs) /* Chain */);
return true;
}
>From 8375e8cb3b3d2c92912326a2c5990287af00d9ad Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 14 Jul 2026 16:11:13 +0000
Subject: [PATCH 7/7] Add -aarch64-enable-sve-multivector-lowering option
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 17 +++++++++++++----
.../AArch64/sve-multivector-load-stores.ll | 6 +++---
2 files changed, 16 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d5f83362e6702..3c42810f3fc17 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -140,6 +140,13 @@ static cl::opt<bool> EnableExtToTBL("aarch64-enable-ext-to-tbl", cl::Hidden,
cl::desc("Combine ext and trunc to TBL"),
cl::init(true));
+static cl::opt<bool> EnableSveMultiVectorLowering(
+ "aarch64-enable-sve-multivector-lowering", cl::Hidden,
+ cl::desc("Enable lowering of oversized SVE loads and stores that are two"
+ "or four times the width of a legeal SVE type to multi-vector "
+ "operations."),
+ cl::init(false));
+
// All of the XOR, OR and CMP use ALU ports, and data dependency will become the
// bottleneck after this transform on high end CPU. So this max leaf node
// limitation is guard cmp+ccmp will be profitable.
@@ -7885,8 +7892,9 @@ SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
}
if (VT.isVector()) {
- if (SDValue Store = tryLowerMultiVectorStore(StoreNode, DAG))
- return Store;
+ if (EnableSveMultiVectorLowering)
+ if (SDValue Store = tryLowerMultiVectorStore(StoreNode, DAG))
+ return Store;
if (useSVEForFixedLengthVectorVT(
VT,
@@ -31502,8 +31510,9 @@ void AArch64TargetLowering::ReplaceNodeResults(
}
if (auto *Load = dyn_cast<LoadSDNode>(N))
- if (tryLowerMultiVectorLoad(Load, Results, DAG))
- return;
+ if (EnableSveMultiVectorLowering)
+ if (tryLowerMultiVectorLoad(Load, Results, DAG))
+ return;
if ((!LoadNode->isVolatile() && !LoadNode->isAtomic()) ||
LoadNode->getMemoryVT() != MVT::i128) {
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll b/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
index 62c2f3d048844..78b26e7d2d2e6 100644
--- a/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-load-stores.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mattr=+sve-b16b16,+sve2p1 < %s -verify-machineinstrs | FileCheck %s --check-prefix=SVE2p1
-; RUN: llc -mattr=+sve-b16b16,+sve2p1 -aarch64-enable-subreg-liveness-tracking < %s -verify-machineinstrs | FileCheck %s --check-prefix=SVE2p1-SL
-; RUN: llc -mattr=+sve-b16b16,+sme2 --force-streaming < %s -verify-machineinstrs | FileCheck %s --check-prefix=SME2
+; RUN: llc -mattr=+sve-b16b16,+sve2p1 -aarch64-enable-sve-multivector-lowering < %s -verify-machineinstrs | FileCheck %s --check-prefix=SVE2p1
+; RUN: llc -mattr=+sve-b16b16,+sve2p1 -aarch64-enable-subreg-liveness-tracking -aarch64-enable-sve-multivector-lowering < %s -verify-machineinstrs | FileCheck %s --check-prefix=SVE2p1-SL
+; RUN: llc -mattr=+sve-b16b16,+sme2 --force-streaming -aarch64-enable-sve-multivector-lowering < %s -verify-machineinstrs | FileCheck %s --check-prefix=SME2
target triple = "aarch64-unknown-linux-gnu"
More information about the llvm-commits
mailing list