[llvm] [LLVM][CodeGen][SVE] Lower to multivector stores instead of splitting… (PR #207397)

Jacob Crawley via llvm-commits llvm-commits at lists.llvm.org
Fri Jul 3 07:15:09 PDT 2026


https://github.com/jacob-crawley created https://github.com/llvm/llvm-project/pull/207397

… them.

Lowers unpredicated stores of scalable vectors that are two or four times the width of a legal SVE type to multi-vector operations when subregister liveness is enabled.

This matches the existing approach for multi-vector load lowering.

>From 6fb9bfb6c7c449ee123844cd837f8b0939d7a7ee Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 2 Jul 2026 15:40:29 +0000
Subject: [PATCH] [LLVM][CodeGen][SVE] Lower to multivector stores instead of
 splitting them.

Lowers unpredicated stores of scalable vectors that are two or four
times the width of a legal SVE type to multi-vector operations when
subregister liveness is enabled.

This matches the existing approach for multi-vector load lowering.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    | 110 ++++
 .../CodeGen/AArch64/sve-multivector-loads.ll  | 610 ++++++++----------
 2 files changed, 383 insertions(+), 337 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e883c8bb5e96e..de51972b202ea 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2106,6 +2106,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::LOAD, MVT::nxv4f64, Custom);
       setOperationAction(ISD::LOAD, MVT::nxv16bf16, Custom);
 
+      // 2x stores
+      setOperationAction(ISD::STORE, MVT::nxv32i8, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv16i16, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv8i32, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv4i64, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv16f16, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv8f32, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv4f64, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv16bf16, Custom);
+
       // 4x loads
       setOperationAction(ISD::LOAD, MVT::nxv64i8, Custom);
       setOperationAction(ISD::LOAD, MVT::nxv32i16, Custom);
@@ -2115,6 +2125,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::LOAD, MVT::nxv16f32, Custom);
       setOperationAction(ISD::LOAD, MVT::nxv8f64, Custom);
       setOperationAction(ISD::LOAD, MVT::nxv32bf16, Custom);
+
+      // 4x stores
+      setOperationAction(ISD::STORE, MVT::nxv64i8, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv32i16, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv16i32, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv8i64, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv32f16, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv16f32, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv8f64, Custom);
+      setOperationAction(ISD::STORE, MVT::nxv32bf16, Custom);
     }
   }
 
@@ -7733,6 +7753,93 @@ static SDValue LowerNTStore(StoreSDNode *StoreNode, EVT VT, EVT MemVT,
   return SDValue();
 }
 
+// Lower scalable vectors that are 2/4 times the width of a legal SVE type to
+// multi-vector operations.
+static SDValue tryLowerMultiVectorStore(StoreSDNode *StoreNode,
+                                        SelectionDAG &DAG) {
+  SDValue Value = StoreNode->getValue();
+  EVT VT = Value.getValueType();
+  EVT MemVT = StoreNode->getMemoryVT();
+
+  if (!StoreNode->isSimple() || !StoreNode->isUnindexed() ||
+      StoreNode->isNonTemporal() || !StoreNode->getOffset().isUndef() ||
+      !VT.isScalableVector() || !VT.isSimple() || VT != MemVT)
+    return SDValue();
+
+  MVT StoreVT = VT.getSimpleVT();
+  MVT RegVT;
+  unsigned IntID;
+  unsigned NumVecs;
+
+  switch (StoreVT.SimpleTy) {
+  default:
+    return SDValue();
+
+  case MVT::nxv32i8:
+  case MVT::nxv16i16:
+  case MVT::nxv8i32:
+  case MVT::nxv4i64:
+  case MVT::nxv16f16:
+  case MVT::nxv8f32:
+  case MVT::nxv4f64:
+  case MVT::nxv16bf16:
+    IntID = Intrinsic::aarch64_sve_st1_pn_x2;
+    NumVecs = 2;
+    RegVT = StoreVT.getHalfNumVectorElementsVT();
+    break;
+  case MVT::nxv64i8:
+  case MVT::nxv32i16:
+  case MVT::nxv16i32:
+  case MVT::nxv8i64:
+  case MVT::nxv32f16:
+  case MVT::nxv16f32:
+  case MVT::nxv8f64:
+  case MVT::nxv32bf16:
+    IntID = Intrinsic::aarch64_sve_st1_pn_x4;
+    NumVecs = 4;
+    RegVT = StoreVT.getHalfNumVectorElementsVT().getHalfNumVectorElementsVT();
+    break;
+  }
+
+  unsigned PredIntID;
+  switch (StoreVT.getScalarSizeInBits()) {
+  default:
+    llvm_unreachable("covered by previous switch");
+  case 8:
+    PredIntID = Intrinsic::aarch64_sve_ptrue_c8;
+    break;
+  case 16:
+    PredIntID = Intrinsic::aarch64_sve_ptrue_c16;
+    break;
+  case 32:
+    PredIntID = Intrinsic::aarch64_sve_ptrue_c32;
+    break;
+  case 64:
+    PredIntID = Intrinsic::aarch64_sve_ptrue_c64;
+    break;
+  }
+
+  SDLoc DL(StoreNode);
+  SDValue PNg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
+                            DAG.getConstant(PredIntID, DL, MVT::i64));
+
+  SmallVector<SDValue, 8> Ops;
+  Ops.push_back(StoreNode->getChain());
+  Ops.push_back(DAG.getConstant(IntID, DL, MVT::i64));
+
+  unsigned RegElts = RegVT.getVectorMinNumElements();
+  for (unsigned i = 0; i != NumVecs; ++i)
+    Ops.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, RegVT, Value,
+                              DAG.getVectorIdxConstant(i * RegElts, DL)));
+
+  Ops.push_back(PNg);
+  Ops.push_back(StoreNode->getBasePtr());
+
+  return DAG.getMemIntrinsicNode(
+      ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
+      StoreNode->getMemoryVT(), StoreNode->getMemOperand());
+}
+
 // Custom lowering for any store, vector or scalar and/or default or with
 // a truncate operations.  Currently only custom lower truncate operation
 // from vector v4i16 to v4i8 or volatile stores of i128.
@@ -7753,6 +7860,9 @@ SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
   }
 
   if (VT.isVector()) {
+    if (SDValue Store = tryLowerMultiVectorStore(StoreNode, DAG))
+      return Store;
+
     if (useSVEForFixedLengthVectorVT(
             VT,
             /*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors()))
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll b/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
index 985326f26a2d5..e7a4ee355f58f 100644
--- a/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
@@ -22,18 +22,18 @@ define void @load_2x_vectors_i8_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    ld1b { z0.b, z1.b }, pn8/z, [x0]
 ; SVE2p1-SL-NEXT:    add z1.b, z1.b, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z0.b, z0.b, #5 // =0x5
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1b { z0.b, z1.b }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_i8_r:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.b
 ; SME2-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x0]
-; SME2-NEXT:    add z24.b, z24.b, #5 // =0x5
-; SME2-NEXT:    add z16.b, z16.b, #5 // =0x5
-; SME2-NEXT:    str z24, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    add z1.b, z1.b, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.b, z0.b, #5 // =0x5
+; SME2-NEXT:    st1b { z0.b, z1.b }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 32 x i8>, ptr %addr
   %b = add <vscale x 32 x i8> %a, splat (i8 5)
@@ -57,25 +57,23 @@ define void @load_2x_vectors_i8_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-LABEL: load_2x_vectors_i8_rr:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.b
-; SVE2p1-SL-NEXT:    ptrue p0.b
 ; SVE2p1-SL-NEXT:    add x8, x0, x1
 ; SVE2p1-SL-NEXT:    ld1b { z0.b, z1.b }, pn8/z, [x0, x1]
-; SVE2p1-SL-NEXT:    add z0.b, z0.b, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.b, z1.b, #5 // =0x5
-; SVE2p1-SL-NEXT:    st1b { z0.b }, p0, [x0, x1]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    add z0.b, z0.b, #5 // =0x5
+; SVE2p1-SL-NEXT:    st1b { z0.b, z1.b }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_i8_rr:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.b
-; SME2-NEXT:    ptrue p0.b
 ; SME2-NEXT:    add x8, x0, x1
 ; SME2-NEXT:    ld1b { z16.b, z24.b }, pn8/z, [x0, x1]
-; SME2-NEXT:    add z16.b, z16.b, #5 // =0x5
-; SME2-NEXT:    add z24.b, z24.b, #5 // =0x5
-; SME2-NEXT:    st1b { z16.b }, p0, [x0, x1]
-; SME2-NEXT:    str z24, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    add z1.b, z1.b, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.b, z0.b, #5 // =0x5
+; SME2-NEXT:    st1b { z0.b, z1.b }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr i8, ptr %base, i64 %idx
   %a = load <vscale x 32 x i8>, ptr %addr
@@ -101,18 +99,18 @@ define void @load_2x_vectors_i16_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    ld1h { z0.h, z1.h }, pn8/z, [x0]
 ; SVE2p1-SL-NEXT:    add z1.h, z1.h, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z0.h, z0.h, #5 // =0x5
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1h { z0.h, z1.h }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_i16_r:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.h
 ; SME2-NEXT:    ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NEXT:    add z24.h, z24.h, #5 // =0x5
-; SME2-NEXT:    add z16.h, z16.h, #5 // =0x5
-; SME2-NEXT:    str z24, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    add z1.h, z1.h, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.h, z0.h, #5 // =0x5
+; SME2-NEXT:    st1h { z0.h, z1.h }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 16 x i16>, ptr %addr
   %b = add <vscale x 16 x i16> %a, splat (i16 5)
@@ -136,25 +134,23 @@ define void @load_2x_vectors_i16_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-LABEL: load_2x_vectors_i16_rr:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.h
-; SVE2p1-SL-NEXT:    ptrue p0.h
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #1
 ; SVE2p1-SL-NEXT:    ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    add z0.h, z0.h, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.h, z1.h, #5 // =0x5
-; SVE2p1-SL-NEXT:    st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    add z0.h, z0.h, #5 // =0x5
+; SVE2p1-SL-NEXT:    st1h { z0.h, z1.h }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_i16_rr:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.h
-; SME2-NEXT:    ptrue p0.h
 ; SME2-NEXT:    add x8, x0, x1, lsl #1
 ; SME2-NEXT:    ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT:    add z16.h, z16.h, #5 // =0x5
-; SME2-NEXT:    add z24.h, z24.h, #5 // =0x5
-; SME2-NEXT:    st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT:    str z24, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    add z1.h, z1.h, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.h, z0.h, #5 // =0x5
+; SME2-NEXT:    st1h { z0.h, z1.h }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr i16, ptr %base, i64 %idx
   %a = load <vscale x 16 x i16>, ptr %addr
@@ -180,18 +176,18 @@ define void @load_2x_vectors_i32_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    ld1w { z0.s, z1.s }, pn8/z, [x0]
 ; SVE2p1-SL-NEXT:    add z1.s, z1.s, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z0.s, z0.s, #5 // =0x5
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1w { z0.s, z1.s }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_i32_r:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.s
 ; SME2-NEXT:    ld1w { z16.s, z24.s }, pn8/z, [x0]
-; SME2-NEXT:    add z24.s, z24.s, #5 // =0x5
-; SME2-NEXT:    add z16.s, z16.s, #5 // =0x5
-; SME2-NEXT:    str z24, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    add z1.s, z1.s, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.s, z0.s, #5 // =0x5
+; SME2-NEXT:    st1w { z0.s, z1.s }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 8 x i32>, ptr %addr
   %b = add <vscale x 8 x i32> %a, splat (i32 5)
@@ -215,25 +211,23 @@ define void @load_2x_vectors_i32_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-LABEL: load_2x_vectors_i32_rr:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.s
-; SVE2p1-SL-NEXT:    ptrue p0.s
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #2
 ; SVE2p1-SL-NEXT:    ld1w { z0.s, z1.s }, pn8/z, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT:    add z0.s, z0.s, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.s, z1.s, #5 // =0x5
-; SVE2p1-SL-NEXT:    st1w { z0.s }, p0, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    add z0.s, z0.s, #5 // =0x5
+; SVE2p1-SL-NEXT:    st1w { z0.s, z1.s }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_i32_rr:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.s
-; SME2-NEXT:    ptrue p0.s
 ; SME2-NEXT:    add x8, x0, x1, lsl #2
 ; SME2-NEXT:    ld1w { z16.s, z24.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT:    add z16.s, z16.s, #5 // =0x5
-; SME2-NEXT:    add z24.s, z24.s, #5 // =0x5
-; SME2-NEXT:    st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NEXT:    str z24, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    add z1.s, z1.s, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.s, z0.s, #5 // =0x5
+; SME2-NEXT:    st1w { z0.s, z1.s }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr i32, ptr %base, i64 %idx
   %a = load <vscale x 8 x i32>, ptr %addr
@@ -259,18 +253,18 @@ define void @load_2x_vectors_i64_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    ld1d { z0.d, z1.d }, pn8/z, [x0]
 ; SVE2p1-SL-NEXT:    add z1.d, z1.d, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z0.d, z0.d, #5 // =0x5
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1d { z0.d, z1.d }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_i64_r:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.d
 ; SME2-NEXT:    ld1d { z16.d, z24.d }, pn8/z, [x0]
-; SME2-NEXT:    add z24.d, z24.d, #5 // =0x5
-; SME2-NEXT:    add z16.d, z16.d, #5 // =0x5
-; SME2-NEXT:    str z24, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    add z1.d, z1.d, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.d, z0.d, #5 // =0x5
+; SME2-NEXT:    st1d { z0.d, z1.d }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 4 x i64>, ptr %addr
   %b = add <vscale x 4 x i64> %a, splat (i64 5)
@@ -294,25 +288,23 @@ define void @load_2x_vectors_i64_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-LABEL: load_2x_vectors_i64_rr:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.d
-; SVE2p1-SL-NEXT:    ptrue p0.d
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #3
 ; SVE2p1-SL-NEXT:    ld1d { z0.d, z1.d }, pn8/z, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT:    add z0.d, z0.d, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.d, z1.d, #5 // =0x5
-; SVE2p1-SL-NEXT:    st1d { z0.d }, p0, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    add z0.d, z0.d, #5 // =0x5
+; SVE2p1-SL-NEXT:    st1d { z0.d, z1.d }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_i64_rr:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.d
-; SME2-NEXT:    ptrue p0.d
 ; SME2-NEXT:    add x8, x0, x1, lsl #3
 ; SME2-NEXT:    ld1d { z16.d, z24.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT:    add z16.d, z16.d, #5 // =0x5
-; SME2-NEXT:    add z24.d, z24.d, #5 // =0x5
-; SME2-NEXT:    st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NEXT:    str z24, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    add z1.d, z1.d, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.d, z0.d, #5 // =0x5
+; SME2-NEXT:    st1d { z0.d, z1.d }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr i64, ptr %base, i64 %idx
   %a = load <vscale x 4 x i64>, ptr %addr
@@ -340,8 +332,7 @@ define void @load_2x_vectors_f16_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    ld1h { z0.h, z1.h }, pn8/z, [x0]
 ; SVE2p1-SL-NEXT:    fadd z1.h, p0/m, z1.h, #1.0
 ; SVE2p1-SL-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1h { z0.h, z1.h }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_f16_r:
@@ -349,10 +340,11 @@ define void @load_2x_vectors_f16_r(ptr %addr) {
 ; SME2-NEXT:    ptrue pn8.h
 ; SME2-NEXT:    ptrue p0.h
 ; SME2-NEXT:    ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NEXT:    fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NEXT:    fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NEXT:    str z24, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    fadd z1.h, p0/m, z1.h, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
+; SME2-NEXT:    st1h { z0.h, z1.h }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 16 x half>, ptr %addr
   %b = fadd <vscale x 16 x half> %a, splat (half 1.0)
@@ -379,10 +371,9 @@ define void @load_2x_vectors_f16_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-NEXT:    ptrue p0.h
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #1
 ; SVE2p1-SL-NEXT:    ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
 ; SVE2p1-SL-NEXT:    fadd z1.h, p0/m, z1.h, #1.0
-; SVE2p1-SL-NEXT:    st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
+; SVE2p1-SL-NEXT:    st1h { z0.h, z1.h }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_f16_rr:
@@ -391,10 +382,11 @@ define void @load_2x_vectors_f16_rr(ptr %base, i64 %idx) {
 ; SME2-NEXT:    ptrue p0.h
 ; SME2-NEXT:    add x8, x0, x1, lsl #1
 ; SME2-NEXT:    ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT:    fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NEXT:    fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NEXT:    st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT:    str z24, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    fadd z1.h, p0/m, z1.h, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
+; SME2-NEXT:    st1h { z0.h, z1.h }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr half, ptr %base, i64 %idx
   %a = load <vscale x 16 x half>, ptr %addr
@@ -422,8 +414,7 @@ define void @load_2x_vectors_f32_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    ld1w { z0.s, z1.s }, pn8/z, [x0]
 ; SVE2p1-SL-NEXT:    fadd z1.s, p0/m, z1.s, #1.0
 ; SVE2p1-SL-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1w { z0.s, z1.s }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_f32_r:
@@ -431,10 +422,11 @@ define void @load_2x_vectors_f32_r(ptr %addr) {
 ; SME2-NEXT:    ptrue pn8.s
 ; SME2-NEXT:    ptrue p0.s
 ; SME2-NEXT:    ld1w { z16.s, z24.s }, pn8/z, [x0]
-; SME2-NEXT:    fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NEXT:    fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NEXT:    str z24, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    fadd z1.s, p0/m, z1.s, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
+; SME2-NEXT:    st1w { z0.s, z1.s }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 8 x float>, ptr %addr
   %b = fadd <vscale x 8 x float> %a, splat (float 1.0)
@@ -461,10 +453,9 @@ define void @load_2x_vectors_f32_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-NEXT:    ptrue p0.s
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #2
 ; SVE2p1-SL-NEXT:    ld1w { z0.s, z1.s }, pn8/z, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
 ; SVE2p1-SL-NEXT:    fadd z1.s, p0/m, z1.s, #1.0
-; SVE2p1-SL-NEXT:    st1w { z0.s }, p0, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
+; SVE2p1-SL-NEXT:    st1w { z0.s, z1.s }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_f32_rr:
@@ -473,10 +464,11 @@ define void @load_2x_vectors_f32_rr(ptr %base, i64 %idx) {
 ; SME2-NEXT:    ptrue p0.s
 ; SME2-NEXT:    add x8, x0, x1, lsl #2
 ; SME2-NEXT:    ld1w { z16.s, z24.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT:    fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NEXT:    fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NEXT:    st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NEXT:    str z24, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    fadd z1.s, p0/m, z1.s, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
+; SME2-NEXT:    st1w { z0.s, z1.s }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr float, ptr %base, i64 %idx
   %a = load <vscale x 8 x float>, ptr %addr
@@ -504,8 +496,7 @@ define void @load_2x_vectors_f64_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    ld1d { z0.d, z1.d }, pn8/z, [x0]
 ; SVE2p1-SL-NEXT:    fadd z1.d, p0/m, z1.d, #1.0
 ; SVE2p1-SL-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1d { z0.d, z1.d }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_f64_r:
@@ -513,10 +504,11 @@ define void @load_2x_vectors_f64_r(ptr %addr) {
 ; SME2-NEXT:    ptrue pn8.d
 ; SME2-NEXT:    ptrue p0.d
 ; SME2-NEXT:    ld1d { z16.d, z24.d }, pn8/z, [x0]
-; SME2-NEXT:    fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NEXT:    fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NEXT:    str z24, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    fadd z1.d, p0/m, z1.d, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
+; SME2-NEXT:    st1d { z0.d, z1.d }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 4 x double>, ptr %addr
   %b = fadd <vscale x 4 x double> %a, splat (double 1.0)
@@ -543,10 +535,9 @@ define void @load_2x_vectors_f64_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-NEXT:    ptrue p0.d
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #3
 ; SVE2p1-SL-NEXT:    ld1d { z0.d, z1.d }, pn8/z, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
 ; SVE2p1-SL-NEXT:    fadd z1.d, p0/m, z1.d, #1.0
-; SVE2p1-SL-NEXT:    st1d { z0.d }, p0, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
+; SVE2p1-SL-NEXT:    st1d { z0.d, z1.d }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_f64_rr:
@@ -555,10 +546,11 @@ define void @load_2x_vectors_f64_rr(ptr %base, i64 %idx) {
 ; SME2-NEXT:    ptrue p0.d
 ; SME2-NEXT:    add x8, x0, x1, lsl #3
 ; SME2-NEXT:    ld1d { z16.d, z24.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT:    fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NEXT:    fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NEXT:    st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NEXT:    str z24, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z1, z24
+; SME2-NEXT:    fadd z1.d, p0/m, z1.d, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
+; SME2-NEXT:    st1d { z0.d, z1.d }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr double, ptr %base, i64 %idx
   %a = load <vscale x 4 x double>, ptr %addr
@@ -582,12 +574,11 @@ define void @load_2x_vectors_bf16_r(ptr %addr) {
 ; SVE2p1-SL-LABEL: load_2x_vectors_bf16_r:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.h
-; SVE2p1-SL-NEXT:    fmov z0.h, #1.87500000
-; SVE2p1-SL-NEXT:    ld1h { z2.h, z3.h }, pn8/z, [x0]
-; SVE2p1-SL-NEXT:    bfadd z1.h, z3.h, z0.h
-; SVE2p1-SL-NEXT:    bfadd z0.h, z2.h, z0.h
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    fmov z2.h, #1.87500000
+; SVE2p1-SL-NEXT:    ld1h { z0.h, z1.h }, pn8/z, [x0]
+; SVE2p1-SL-NEXT:    bfadd z1.h, z1.h, z2.h
+; SVE2p1-SL-NEXT:    bfadd z0.h, z0.h, z2.h
+; SVE2p1-SL-NEXT:    st1h { z0.h, z1.h }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_bf16_r:
@@ -597,8 +588,7 @@ define void @load_2x_vectors_bf16_r(ptr %addr) {
 ; SME2-NEXT:    ld1h { z16.h, z24.h }, pn8/z, [x0]
 ; SME2-NEXT:    bfadd z1.h, z24.h, z0.h
 ; SME2-NEXT:    bfadd z0.h, z16.h, z0.h
-; SME2-NEXT:    str z1, [x0, #1, mul vl]
-; SME2-NEXT:    str z0, [x0]
+; SME2-NEXT:    st1h { z0.h, z1.h }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 16 x bfloat>, ptr %addr
   %b = fadd <vscale x 16 x bfloat> %a, splat (bfloat 1.0)
@@ -623,14 +613,12 @@ define void @load_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-LABEL: load_2x_vectors_bf16_rr:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.h
-; SVE2p1-SL-NEXT:    fmov z0.h, #1.87500000
+; SVE2p1-SL-NEXT:    fmov z2.h, #1.87500000
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #1
-; SVE2p1-SL-NEXT:    ld1h { z2.h, z3.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    ptrue p0.h
-; SVE2p1-SL-NEXT:    bfadd z1.h, z2.h, z0.h
-; SVE2p1-SL-NEXT:    bfadd z0.h, z3.h, z0.h
-; SVE2p1-SL-NEXT:    st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    str z0, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]
+; SVE2p1-SL-NEXT:    bfadd z1.h, z1.h, z2.h
+; SVE2p1-SL-NEXT:    bfadd z0.h, z0.h, z2.h
+; SVE2p1-SL-NEXT:    st1h { z0.h, z1.h }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_2x_vectors_bf16_rr:
@@ -639,11 +627,9 @@ define void @load_2x_vectors_bf16_rr(ptr %base, i64 %idx) {
 ; SME2-NEXT:    fmov z0.h, #1.87500000
 ; SME2-NEXT:    add x8, x0, x1, lsl #1
 ; SME2-NEXT:    ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT:    ptrue p0.h
-; SME2-NEXT:    bfadd z1.h, z16.h, z0.h
-; SME2-NEXT:    bfadd z0.h, z24.h, z0.h
-; SME2-NEXT:    st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT:    str z0, [x8, #1, mul vl]
+; SME2-NEXT:    bfadd z1.h, z24.h, z0.h
+; SME2-NEXT:    bfadd z0.h, z16.h, z0.h
+; SME2-NEXT:    st1h { z0.h, z1.h }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr bfloat, ptr %base, i64 %idx
   %a = load <vscale x 16 x bfloat>, ptr %addr
@@ -679,24 +665,22 @@ define void @load_4x_vectors_i8_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    add z2.b, z2.b, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.b, z1.b, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z0.b, z0.b, #5 // =0x5
-; SVE2p1-SL-NEXT:    str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1b { z0.b - z3.b }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_i8_r:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.b
 ; SME2-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
-; SME2-NEXT:    add z28.b, z28.b, #5 // =0x5
-; SME2-NEXT:    add z24.b, z24.b, #5 // =0x5
-; SME2-NEXT:    add z20.b, z20.b, #5 // =0x5
-; SME2-NEXT:    add z16.b, z16.b, #5 // =0x5
-; SME2-NEXT:    str z28, [x0, #3, mul vl]
-; SME2-NEXT:    str z24, [x0, #2, mul vl]
-; SME2-NEXT:    str z20, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    add z3.b, z3.b, #5 // =0x5
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    add z2.b, z2.b, #5 // =0x5
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    add z1.b, z1.b, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.b, z0.b, #5 // =0x5
+; SME2-NEXT:    st1b { z0.b - z3.b }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 64 x i8>, ptr %addr
   %b = add <vscale x 64 x i8> %a, splat (i8 5)
@@ -726,33 +710,29 @@ define void @load_4x_vectors_i8_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-LABEL: load_4x_vectors_i8_rr:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.b
-; SVE2p1-SL-NEXT:    ptrue p0.b
 ; SVE2p1-SL-NEXT:    add x8, x0, x1
 ; SVE2p1-SL-NEXT:    ld1b { z0.b - z3.b }, pn8/z, [x0, x1]
-; SVE2p1-SL-NEXT:    add z0.b, z0.b, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z3.b, z3.b, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z2.b, z2.b, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.b, z1.b, #5 // =0x5
-; SVE2p1-SL-NEXT:    st1b { z0.b }, p0, [x0, x1]
-; SVE2p1-SL-NEXT:    str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    add z0.b, z0.b, #5 // =0x5
+; SVE2p1-SL-NEXT:    st1b { z0.b - z3.b }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_i8_rr:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.b
-; SME2-NEXT:    ptrue p0.b
 ; SME2-NEXT:    add x8, x0, x1
 ; SME2-NEXT:    ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0, x1]
-; SME2-NEXT:    add z16.b, z16.b, #5 // =0x5
-; SME2-NEXT:    add z28.b, z28.b, #5 // =0x5
-; SME2-NEXT:    add z24.b, z24.b, #5 // =0x5
-; SME2-NEXT:    add z20.b, z20.b, #5 // =0x5
-; SME2-NEXT:    st1b { z16.b }, p0, [x0, x1]
-; SME2-NEXT:    str z28, [x8, #3, mul vl]
-; SME2-NEXT:    str z24, [x8, #2, mul vl]
-; SME2-NEXT:    str z20, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    add z3.b, z3.b, #5 // =0x5
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    add z2.b, z2.b, #5 // =0x5
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    add z1.b, z1.b, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.b, z0.b, #5 // =0x5
+; SME2-NEXT:    st1b { z0.b - z3.b }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr i8, ptr %base, i64 %idx
   %a = load <vscale x 64 x i8>, ptr %addr
@@ -786,24 +766,22 @@ define void @load_4x_vectors_i16_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    add z2.h, z2.h, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.h, z1.h, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z0.h, z0.h, #5 // =0x5
-; SVE2p1-SL-NEXT:    str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1h { z0.h - z3.h }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_i16_r:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.h
 ; SME2-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NEXT:    add z28.h, z28.h, #5 // =0x5
-; SME2-NEXT:    add z24.h, z24.h, #5 // =0x5
-; SME2-NEXT:    add z20.h, z20.h, #5 // =0x5
-; SME2-NEXT:    add z16.h, z16.h, #5 // =0x5
-; SME2-NEXT:    str z28, [x0, #3, mul vl]
-; SME2-NEXT:    str z24, [x0, #2, mul vl]
-; SME2-NEXT:    str z20, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    add z3.h, z3.h, #5 // =0x5
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    add z2.h, z2.h, #5 // =0x5
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    add z1.h, z1.h, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.h, z0.h, #5 // =0x5
+; SME2-NEXT:    st1h { z0.h - z3.h }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 32 x i16>, ptr %addr
   %b = add <vscale x 32 x i16> %a, splat (i16 5)
@@ -833,33 +811,29 @@ define void @load_4x_vectors_i16_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-LABEL: load_4x_vectors_i16_rr:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.h
-; SVE2p1-SL-NEXT:    ptrue p0.h
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #1
 ; SVE2p1-SL-NEXT:    ld1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    add z0.h, z0.h, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z3.h, z3.h, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z2.h, z2.h, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.h, z1.h, #5 // =0x5
-; SVE2p1-SL-NEXT:    st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    add z0.h, z0.h, #5 // =0x5
+; SVE2p1-SL-NEXT:    st1h { z0.h - z3.h }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_i16_rr:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.h
-; SME2-NEXT:    ptrue p0.h
 ; SME2-NEXT:    add x8, x0, x1, lsl #1
 ; SME2-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT:    add z16.h, z16.h, #5 // =0x5
-; SME2-NEXT:    add z28.h, z28.h, #5 // =0x5
-; SME2-NEXT:    add z24.h, z24.h, #5 // =0x5
-; SME2-NEXT:    add z20.h, z20.h, #5 // =0x5
-; SME2-NEXT:    st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT:    str z28, [x8, #3, mul vl]
-; SME2-NEXT:    str z24, [x8, #2, mul vl]
-; SME2-NEXT:    str z20, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    add z3.h, z3.h, #5 // =0x5
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    add z2.h, z2.h, #5 // =0x5
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    add z1.h, z1.h, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.h, z0.h, #5 // =0x5
+; SME2-NEXT:    st1h { z0.h - z3.h }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr i16, ptr %base, i64 %idx
   %a = load <vscale x 32 x i16>, ptr %addr
@@ -893,24 +867,22 @@ define void @load_4x_vectors_i32_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    add z2.s, z2.s, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.s, z1.s, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z0.s, z0.s, #5 // =0x5
-; SVE2p1-SL-NEXT:    str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1w { z0.s - z3.s }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_i32_r:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.s
 ; SME2-NEXT:    ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0]
-; SME2-NEXT:    add z28.s, z28.s, #5 // =0x5
-; SME2-NEXT:    add z24.s, z24.s, #5 // =0x5
-; SME2-NEXT:    add z20.s, z20.s, #5 // =0x5
-; SME2-NEXT:    add z16.s, z16.s, #5 // =0x5
-; SME2-NEXT:    str z28, [x0, #3, mul vl]
-; SME2-NEXT:    str z24, [x0, #2, mul vl]
-; SME2-NEXT:    str z20, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    add z3.s, z3.s, #5 // =0x5
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    add z2.s, z2.s, #5 // =0x5
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    add z1.s, z1.s, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.s, z0.s, #5 // =0x5
+; SME2-NEXT:    st1w { z0.s - z3.s }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 16 x i32>, ptr %addr
   %b = add <vscale x 16 x i32> %a, splat (i32 5)
@@ -940,33 +912,29 @@ define void @load_4x_vectors_i32_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-LABEL: load_4x_vectors_i32_rr:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.s
-; SVE2p1-SL-NEXT:    ptrue p0.s
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #2
 ; SVE2p1-SL-NEXT:    ld1w { z0.s - z3.s }, pn8/z, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT:    add z0.s, z0.s, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z3.s, z3.s, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z2.s, z2.s, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.s, z1.s, #5 // =0x5
-; SVE2p1-SL-NEXT:    st1w { z0.s }, p0, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT:    str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    add z0.s, z0.s, #5 // =0x5
+; SVE2p1-SL-NEXT:    st1w { z0.s - z3.s }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_i32_rr:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.s
-; SME2-NEXT:    ptrue p0.s
 ; SME2-NEXT:    add x8, x0, x1, lsl #2
 ; SME2-NEXT:    ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT:    add z16.s, z16.s, #5 // =0x5
-; SME2-NEXT:    add z28.s, z28.s, #5 // =0x5
-; SME2-NEXT:    add z24.s, z24.s, #5 // =0x5
-; SME2-NEXT:    add z20.s, z20.s, #5 // =0x5
-; SME2-NEXT:    st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NEXT:    str z28, [x8, #3, mul vl]
-; SME2-NEXT:    str z24, [x8, #2, mul vl]
-; SME2-NEXT:    str z20, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    add z3.s, z3.s, #5 // =0x5
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    add z2.s, z2.s, #5 // =0x5
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    add z1.s, z1.s, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.s, z0.s, #5 // =0x5
+; SME2-NEXT:    st1w { z0.s - z3.s }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr i32, ptr %base, i64 %idx
   %a = load <vscale x 16 x i32>, ptr %addr
@@ -1000,24 +968,22 @@ define void @load_4x_vectors_i64_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    add z2.d, z2.d, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.d, z1.d, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z0.d, z0.d, #5 // =0x5
-; SVE2p1-SL-NEXT:    str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1d { z0.d - z3.d }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_i64_r:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.d
 ; SME2-NEXT:    ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0]
-; SME2-NEXT:    add z28.d, z28.d, #5 // =0x5
-; SME2-NEXT:    add z24.d, z24.d, #5 // =0x5
-; SME2-NEXT:    add z20.d, z20.d, #5 // =0x5
-; SME2-NEXT:    add z16.d, z16.d, #5 // =0x5
-; SME2-NEXT:    str z28, [x0, #3, mul vl]
-; SME2-NEXT:    str z24, [x0, #2, mul vl]
-; SME2-NEXT:    str z20, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    add z3.d, z3.d, #5 // =0x5
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    add z2.d, z2.d, #5 // =0x5
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    add z1.d, z1.d, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.d, z0.d, #5 // =0x5
+; SME2-NEXT:    st1d { z0.d - z3.d }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 8 x i64>, ptr %addr
   %b = add <vscale x 8 x i64> %a, splat (i64 5)
@@ -1047,33 +1013,29 @@ define void @load_4x_vectors_i64_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-LABEL: load_4x_vectors_i64_rr:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.d
-; SVE2p1-SL-NEXT:    ptrue p0.d
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #3
 ; SVE2p1-SL-NEXT:    ld1d { z0.d - z3.d }, pn8/z, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT:    add z0.d, z0.d, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z3.d, z3.d, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z2.d, z2.d, #5 // =0x5
 ; SVE2p1-SL-NEXT:    add z1.d, z1.d, #5 // =0x5
-; SVE2p1-SL-NEXT:    st1d { z0.d }, p0, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT:    str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    add z0.d, z0.d, #5 // =0x5
+; SVE2p1-SL-NEXT:    st1d { z0.d - z3.d }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_i64_rr:
 ; SME2:       // %bb.0:
 ; SME2-NEXT:    ptrue pn8.d
-; SME2-NEXT:    ptrue p0.d
 ; SME2-NEXT:    add x8, x0, x1, lsl #3
 ; SME2-NEXT:    ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT:    add z16.d, z16.d, #5 // =0x5
-; SME2-NEXT:    add z28.d, z28.d, #5 // =0x5
-; SME2-NEXT:    add z24.d, z24.d, #5 // =0x5
-; SME2-NEXT:    add z20.d, z20.d, #5 // =0x5
-; SME2-NEXT:    st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NEXT:    str z28, [x8, #3, mul vl]
-; SME2-NEXT:    str z24, [x8, #2, mul vl]
-; SME2-NEXT:    str z20, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    add z3.d, z3.d, #5 // =0x5
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    add z2.d, z2.d, #5 // =0x5
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    add z1.d, z1.d, #5 // =0x5
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    add z0.d, z0.d, #5 // =0x5
+; SME2-NEXT:    st1d { z0.d - z3.d }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr i64, ptr %base, i64 %idx
   %a = load <vscale x 8 x i64>, ptr %addr
@@ -1109,10 +1071,7 @@ define void @load_4x_vectors_f16_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    fadd z2.h, p0/m, z2.h, #1.0
 ; SVE2p1-SL-NEXT:    fadd z1.h, p0/m, z1.h, #1.0
 ; SVE2p1-SL-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
-; SVE2p1-SL-NEXT:    str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1h { z0.h - z3.h }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_f16_r:
@@ -1120,14 +1079,15 @@ define void @load_4x_vectors_f16_r(ptr %addr) {
 ; SME2-NEXT:    ptrue pn8.h
 ; SME2-NEXT:    ptrue p0.h
 ; SME2-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NEXT:    fadd z28.h, p0/m, z28.h, #1.0
-; SME2-NEXT:    fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NEXT:    fadd z20.h, p0/m, z20.h, #1.0
-; SME2-NEXT:    fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NEXT:    str z28, [x0, #3, mul vl]
-; SME2-NEXT:    str z24, [x0, #2, mul vl]
-; SME2-NEXT:    str z20, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    fadd z3.h, p0/m, z3.h, #1.0
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    fadd z2.h, p0/m, z2.h, #1.0
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    fadd z1.h, p0/m, z1.h, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
+; SME2-NEXT:    st1h { z0.h - z3.h }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 32 x half>, ptr %addr
   %b = fadd <vscale x 32 x half> %a, splat (half 1.0)
@@ -1160,14 +1120,11 @@ define void @load_4x_vectors_f16_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-NEXT:    ptrue p0.h
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #1
 ; SVE2p1-SL-NEXT:    ld1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
 ; SVE2p1-SL-NEXT:    fadd z3.h, p0/m, z3.h, #1.0
 ; SVE2p1-SL-NEXT:    fadd z2.h, p0/m, z2.h, #1.0
 ; SVE2p1-SL-NEXT:    fadd z1.h, p0/m, z1.h, #1.0
-; SVE2p1-SL-NEXT:    st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
+; SVE2p1-SL-NEXT:    st1h { z0.h - z3.h }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_f16_rr:
@@ -1176,14 +1133,15 @@ define void @load_4x_vectors_f16_rr(ptr %base, i64 %idx) {
 ; SME2-NEXT:    ptrue p0.h
 ; SME2-NEXT:    add x8, x0, x1, lsl #1
 ; SME2-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT:    fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NEXT:    fadd z28.h, p0/m, z28.h, #1.0
-; SME2-NEXT:    fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NEXT:    fadd z20.h, p0/m, z20.h, #1.0
-; SME2-NEXT:    st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT:    str z28, [x8, #3, mul vl]
-; SME2-NEXT:    str z24, [x8, #2, mul vl]
-; SME2-NEXT:    str z20, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    fadd z3.h, p0/m, z3.h, #1.0
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    fadd z2.h, p0/m, z2.h, #1.0
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    fadd z1.h, p0/m, z1.h, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.h, p0/m, z0.h, #1.0
+; SME2-NEXT:    st1h { z0.h - z3.h }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr half, ptr %base, i64 %idx
   %a = load <vscale x 32 x half>, ptr %addr
@@ -1219,10 +1177,7 @@ define void @load_4x_vectors_f32_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    fadd z2.s, p0/m, z2.s, #1.0
 ; SVE2p1-SL-NEXT:    fadd z1.s, p0/m, z1.s, #1.0
 ; SVE2p1-SL-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
-; SVE2p1-SL-NEXT:    str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1w { z0.s - z3.s }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_f32_r:
@@ -1230,14 +1185,15 @@ define void @load_4x_vectors_f32_r(ptr %addr) {
 ; SME2-NEXT:    ptrue pn8.s
 ; SME2-NEXT:    ptrue p0.s
 ; SME2-NEXT:    ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0]
-; SME2-NEXT:    fadd z28.s, p0/m, z28.s, #1.0
-; SME2-NEXT:    fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NEXT:    fadd z20.s, p0/m, z20.s, #1.0
-; SME2-NEXT:    fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NEXT:    str z28, [x0, #3, mul vl]
-; SME2-NEXT:    str z24, [x0, #2, mul vl]
-; SME2-NEXT:    str z20, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    fadd z3.s, p0/m, z3.s, #1.0
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    fadd z2.s, p0/m, z2.s, #1.0
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    fadd z1.s, p0/m, z1.s, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
+; SME2-NEXT:    st1w { z0.s - z3.s }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 16 x float>, ptr %addr
   %b = fadd <vscale x 16 x float> %a, splat (float 1.0)
@@ -1270,14 +1226,11 @@ define void @load_4x_vectors_f32_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-NEXT:    ptrue p0.s
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #2
 ; SVE2p1-SL-NEXT:    ld1w { z0.s - z3.s }, pn8/z, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
 ; SVE2p1-SL-NEXT:    fadd z3.s, p0/m, z3.s, #1.0
 ; SVE2p1-SL-NEXT:    fadd z2.s, p0/m, z2.s, #1.0
 ; SVE2p1-SL-NEXT:    fadd z1.s, p0/m, z1.s, #1.0
-; SVE2p1-SL-NEXT:    st1w { z0.s }, p0, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT:    str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
+; SVE2p1-SL-NEXT:    st1w { z0.s - z3.s }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_f32_rr:
@@ -1286,14 +1239,15 @@ define void @load_4x_vectors_f32_rr(ptr %base, i64 %idx) {
 ; SME2-NEXT:    ptrue p0.s
 ; SME2-NEXT:    add x8, x0, x1, lsl #2
 ; SME2-NEXT:    ld1w { z16.s, z20.s, z24.s, z28.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT:    fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NEXT:    fadd z28.s, p0/m, z28.s, #1.0
-; SME2-NEXT:    fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NEXT:    fadd z20.s, p0/m, z20.s, #1.0
-; SME2-NEXT:    st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NEXT:    str z28, [x8, #3, mul vl]
-; SME2-NEXT:    str z24, [x8, #2, mul vl]
-; SME2-NEXT:    str z20, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    fadd z3.s, p0/m, z3.s, #1.0
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    fadd z2.s, p0/m, z2.s, #1.0
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    fadd z1.s, p0/m, z1.s, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.s, p0/m, z0.s, #1.0
+; SME2-NEXT:    st1w { z0.s - z3.s }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr float, ptr %base, i64 %idx
   %a = load <vscale x 16 x float>, ptr %addr
@@ -1329,10 +1283,7 @@ define void @load_4x_vectors_f64_r(ptr %addr) {
 ; SVE2p1-SL-NEXT:    fadd z2.d, p0/m, z2.d, #1.0
 ; SVE2p1-SL-NEXT:    fadd z1.d, p0/m, z1.d, #1.0
 ; SVE2p1-SL-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
-; SVE2p1-SL-NEXT:    str z3, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    st1d { z0.d - z3.d }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_f64_r:
@@ -1340,14 +1291,15 @@ define void @load_4x_vectors_f64_r(ptr %addr) {
 ; SME2-NEXT:    ptrue pn8.d
 ; SME2-NEXT:    ptrue p0.d
 ; SME2-NEXT:    ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0]
-; SME2-NEXT:    fadd z28.d, p0/m, z28.d, #1.0
-; SME2-NEXT:    fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NEXT:    fadd z20.d, p0/m, z20.d, #1.0
-; SME2-NEXT:    fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NEXT:    str z28, [x0, #3, mul vl]
-; SME2-NEXT:    str z24, [x0, #2, mul vl]
-; SME2-NEXT:    str z20, [x0, #1, mul vl]
-; SME2-NEXT:    str z16, [x0]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    fadd z3.d, p0/m, z3.d, #1.0
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    fadd z2.d, p0/m, z2.d, #1.0
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    fadd z1.d, p0/m, z1.d, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
+; SME2-NEXT:    st1d { z0.d - z3.d }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 8 x double>, ptr %addr
   %b = fadd <vscale x 8 x double> %a, splat (double 1.0)
@@ -1380,14 +1332,11 @@ define void @load_4x_vectors_f64_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-NEXT:    ptrue p0.d
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #3
 ; SVE2p1-SL-NEXT:    ld1d { z0.d - z3.d }, pn8/z, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
 ; SVE2p1-SL-NEXT:    fadd z3.d, p0/m, z3.d, #1.0
 ; SVE2p1-SL-NEXT:    fadd z2.d, p0/m, z2.d, #1.0
 ; SVE2p1-SL-NEXT:    fadd z1.d, p0/m, z1.d, #1.0
-; SVE2p1-SL-NEXT:    st1d { z0.d }, p0, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT:    str z3, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z2, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
+; SVE2p1-SL-NEXT:    st1d { z0.d - z3.d }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_f64_rr:
@@ -1396,14 +1345,15 @@ define void @load_4x_vectors_f64_rr(ptr %base, i64 %idx) {
 ; SME2-NEXT:    ptrue p0.d
 ; SME2-NEXT:    add x8, x0, x1, lsl #3
 ; SME2-NEXT:    ld1d { z16.d, z20.d, z24.d, z28.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT:    fadd z16.d, p0/m, z16.d, #1.0
-; SME2-NEXT:    fadd z28.d, p0/m, z28.d, #1.0
-; SME2-NEXT:    fadd z24.d, p0/m, z24.d, #1.0
-; SME2-NEXT:    fadd z20.d, p0/m, z20.d, #1.0
-; SME2-NEXT:    st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NEXT:    str z28, [x8, #3, mul vl]
-; SME2-NEXT:    str z24, [x8, #2, mul vl]
-; SME2-NEXT:    str z20, [x8, #1, mul vl]
+; SME2-NEXT:    movprfx z3, z28
+; SME2-NEXT:    fadd z3.d, p0/m, z3.d, #1.0
+; SME2-NEXT:    movprfx z2, z24
+; SME2-NEXT:    fadd z2.d, p0/m, z2.d, #1.0
+; SME2-NEXT:    movprfx z1, z20
+; SME2-NEXT:    fadd z1.d, p0/m, z1.d, #1.0
+; SME2-NEXT:    movprfx z0, z16
+; SME2-NEXT:    fadd z0.d, p0/m, z0.d, #1.0
+; SME2-NEXT:    st1d { z0.d - z3.d }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr double, ptr %base, i64 %idx
   %a = load <vscale x 8 x double>, ptr %addr
@@ -1433,16 +1383,13 @@ define void @load_4x_vectors_bf16_r(ptr %addr) {
 ; SVE2p1-SL-LABEL: load_4x_vectors_bf16_r:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.h
-; SVE2p1-SL-NEXT:    fmov z0.h, #1.87500000
-; SVE2p1-SL-NEXT:    ld1h { z4.h - z7.h }, pn8/z, [x0]
-; SVE2p1-SL-NEXT:    bfadd z1.h, z7.h, z0.h
-; SVE2p1-SL-NEXT:    bfadd z2.h, z6.h, z0.h
-; SVE2p1-SL-NEXT:    str z1, [x0, #3, mul vl]
-; SVE2p1-SL-NEXT:    bfadd z1.h, z5.h, z0.h
-; SVE2p1-SL-NEXT:    bfadd z0.h, z4.h, z0.h
-; SVE2p1-SL-NEXT:    str z2, [x0, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x0]
+; SVE2p1-SL-NEXT:    fmov z4.h, #1.87500000
+; SVE2p1-SL-NEXT:    ld1h { z0.h - z3.h }, pn8/z, [x0]
+; SVE2p1-SL-NEXT:    bfadd z3.h, z3.h, z4.h
+; SVE2p1-SL-NEXT:    bfadd z2.h, z2.h, z4.h
+; SVE2p1-SL-NEXT:    bfadd z1.h, z1.h, z4.h
+; SVE2p1-SL-NEXT:    bfadd z0.h, z0.h, z4.h
+; SVE2p1-SL-NEXT:    st1h { z0.h - z3.h }, pn8, [x0]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_bf16_r:
@@ -1450,14 +1397,11 @@ define void @load_4x_vectors_bf16_r(ptr %addr) {
 ; SME2-NEXT:    ptrue pn8.h
 ; SME2-NEXT:    fmov z0.h, #1.87500000
 ; SME2-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0]
-; SME2-NEXT:    bfadd z1.h, z28.h, z0.h
+; SME2-NEXT:    bfadd z3.h, z28.h, z0.h
 ; SME2-NEXT:    bfadd z2.h, z24.h, z0.h
-; SME2-NEXT:    str z1, [x0, #3, mul vl]
 ; SME2-NEXT:    bfadd z1.h, z20.h, z0.h
 ; SME2-NEXT:    bfadd z0.h, z16.h, z0.h
-; SME2-NEXT:    str z2, [x0, #2, mul vl]
-; SME2-NEXT:    str z1, [x0, #1, mul vl]
-; SME2-NEXT:    str z0, [x0]
+; SME2-NEXT:    st1h { z0.h - z3.h }, pn8, [x0]
 ; SME2-NEXT:    ret
   %a = load <vscale x 32 x bfloat>, ptr %addr
   %b = fadd <vscale x 32 x bfloat> %a, splat (bfloat 1.0)
@@ -1488,18 +1432,14 @@ define void @load_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
 ; SVE2p1-SL-LABEL: load_4x_vectors_bf16_rr:
 ; SVE2p1-SL:       // %bb.0:
 ; SVE2p1-SL-NEXT:    ptrue pn8.h
-; SVE2p1-SL-NEXT:    fmov z0.h, #1.87500000
+; SVE2p1-SL-NEXT:    fmov z4.h, #1.87500000
 ; SVE2p1-SL-NEXT:    add x8, x0, x1, lsl #1
-; SVE2p1-SL-NEXT:    ld1h { z4.h - z7.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    ptrue p0.h
-; SVE2p1-SL-NEXT:    bfadd z1.h, z4.h, z0.h
-; SVE2p1-SL-NEXT:    bfadd z2.h, z7.h, z0.h
-; SVE2p1-SL-NEXT:    st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT:    bfadd z1.h, z6.h, z0.h
-; SVE2p1-SL-NEXT:    bfadd z0.h, z5.h, z0.h
-; SVE2p1-SL-NEXT:    str z2, [x8, #3, mul vl]
-; SVE2p1-SL-NEXT:    str z1, [x8, #2, mul vl]
-; SVE2p1-SL-NEXT:    str z0, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT:    ld1h { z0.h - z3.h }, pn8/z, [x0, x1, lsl #1]
+; SVE2p1-SL-NEXT:    bfadd z3.h, z3.h, z4.h
+; SVE2p1-SL-NEXT:    bfadd z2.h, z2.h, z4.h
+; SVE2p1-SL-NEXT:    bfadd z1.h, z1.h, z4.h
+; SVE2p1-SL-NEXT:    bfadd z0.h, z0.h, z4.h
+; SVE2p1-SL-NEXT:    st1h { z0.h - z3.h }, pn8, [x8]
 ; SVE2p1-SL-NEXT:    ret
 ;
 ; SME2-LABEL: load_4x_vectors_bf16_rr:
@@ -1508,15 +1448,11 @@ define void @load_4x_vectors_bf16_rr(ptr %base, i64 %idx) {
 ; SME2-NEXT:    fmov z0.h, #1.87500000
 ; SME2-NEXT:    add x8, x0, x1, lsl #1
 ; SME2-NEXT:    ld1h { z16.h, z20.h, z24.h, z28.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT:    ptrue p0.h
-; SME2-NEXT:    bfadd z1.h, z16.h, z0.h
-; SME2-NEXT:    bfadd z2.h, z28.h, z0.h
-; SME2-NEXT:    st1h { z1.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT:    bfadd z1.h, z24.h, z0.h
-; SME2-NEXT:    bfadd z0.h, z20.h, z0.h
-; SME2-NEXT:    str z2, [x8, #3, mul vl]
-; SME2-NEXT:    str z1, [x8, #2, mul vl]
-; SME2-NEXT:    str z0, [x8, #1, mul vl]
+; SME2-NEXT:    bfadd z3.h, z28.h, z0.h
+; SME2-NEXT:    bfadd z2.h, z24.h, z0.h
+; SME2-NEXT:    bfadd z1.h, z20.h, z0.h
+; SME2-NEXT:    bfadd z0.h, z16.h, z0.h
+; SME2-NEXT:    st1h { z0.h - z3.h }, pn8, [x8]
 ; SME2-NEXT:    ret
   %addr = getelementptr bfloat, ptr %base, i64 %idx
   %a = load <vscale x 32 x bfloat>, ptr %addr



More information about the llvm-commits mailing list