[llvm] [LLVM][CodeGen][SVE] Lower to multivector stores (PR #207397)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 3 08:46:49 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Jacob Crawley (jacob-crawley)
<details>
<summary>Changes</summary>
Lowers unpredicated stores of scalable vectors that are two or four times the width of a legal SVE type to multi-vector operations when subregister liveness is enabled instead of splitting them.
This matches the existing approach for multi-vector load lowering.
---
Patch is 61.70 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/207397.diff
3 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+110)
- (modified) llvm/test/CodeGen/AArch64/sve-multivector-loads.ll (+273-337)
- (modified) llvm/test/CodeGen/AArch64/sve-vector-interleave.ll (+144-60)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e883c8bb5e96e..de51972b202ea 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2106,6 +2106,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::nxv4f64, Custom);
setOperationAction(ISD::LOAD, MVT::nxv16bf16, Custom);
+ // 2x stores
+ setOperationAction(ISD::STORE, MVT::nxv32i8, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv16i16, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv8i32, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv4i64, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv16f16, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv8f32, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv4f64, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv16bf16, Custom);
+
// 4x loads
setOperationAction(ISD::LOAD, MVT::nxv64i8, Custom);
setOperationAction(ISD::LOAD, MVT::nxv32i16, Custom);
@@ -2115,6 +2125,16 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::nxv16f32, Custom);
setOperationAction(ISD::LOAD, MVT::nxv8f64, Custom);
setOperationAction(ISD::LOAD, MVT::nxv32bf16, Custom);
+
+ // 4x stores
+ setOperationAction(ISD::STORE, MVT::nxv64i8, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv32i16, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv16i32, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv8i64, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv32f16, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv16f32, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv8f64, Custom);
+ setOperationAction(ISD::STORE, MVT::nxv32bf16, Custom);
}
}
@@ -7733,6 +7753,93 @@ static SDValue LowerNTStore(StoreSDNode *StoreNode, EVT VT, EVT MemVT,
return SDValue();
}
+// Lower scalable vectors that are 2/4 times the width of a legal SVE type to
+// multi-vector operations.
+static SDValue tryLowerMultiVectorStore(StoreSDNode *StoreNode,
+ SelectionDAG &DAG) {
+ SDValue Value = StoreNode->getValue();
+ EVT VT = Value.getValueType();
+ EVT MemVT = StoreNode->getMemoryVT();
+
+ if (!StoreNode->isSimple() || !StoreNode->isUnindexed() ||
+ StoreNode->isNonTemporal() || !StoreNode->getOffset().isUndef() ||
+ !VT.isScalableVector() || !VT.isSimple() || VT != MemVT)
+ return SDValue();
+
+ MVT StoreVT = VT.getSimpleVT();
+ MVT RegVT;
+ unsigned IntID;
+ unsigned NumVecs;
+
+ switch (StoreVT.SimpleTy) {
+ default:
+ return SDValue();
+
+ case MVT::nxv32i8:
+ case MVT::nxv16i16:
+ case MVT::nxv8i32:
+ case MVT::nxv4i64:
+ case MVT::nxv16f16:
+ case MVT::nxv8f32:
+ case MVT::nxv4f64:
+ case MVT::nxv16bf16:
+ IntID = Intrinsic::aarch64_sve_st1_pn_x2;
+ NumVecs = 2;
+ RegVT = StoreVT.getHalfNumVectorElementsVT();
+ break;
+ case MVT::nxv64i8:
+ case MVT::nxv32i16:
+ case MVT::nxv16i32:
+ case MVT::nxv8i64:
+ case MVT::nxv32f16:
+ case MVT::nxv16f32:
+ case MVT::nxv8f64:
+ case MVT::nxv32bf16:
+ IntID = Intrinsic::aarch64_sve_st1_pn_x4;
+ NumVecs = 4;
+ RegVT = StoreVT.getHalfNumVectorElementsVT().getHalfNumVectorElementsVT();
+ break;
+ }
+
+ unsigned PredIntID;
+ switch (StoreVT.getScalarSizeInBits()) {
+ default:
+ llvm_unreachable("covered by previous switch");
+ case 8:
+ PredIntID = Intrinsic::aarch64_sve_ptrue_c8;
+ break;
+ case 16:
+ PredIntID = Intrinsic::aarch64_sve_ptrue_c16;
+ break;
+ case 32:
+ PredIntID = Intrinsic::aarch64_sve_ptrue_c32;
+ break;
+ case 64:
+ PredIntID = Intrinsic::aarch64_sve_ptrue_c64;
+ break;
+ }
+
+ SDLoc DL(StoreNode);
+ SDValue PNg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::aarch64svcount,
+ DAG.getConstant(PredIntID, DL, MVT::i64));
+
+ SmallVector<SDValue, 8> Ops;
+ Ops.push_back(StoreNode->getChain());
+ Ops.push_back(DAG.getConstant(IntID, DL, MVT::i64));
+
+ unsigned RegElts = RegVT.getVectorMinNumElements();
+ for (unsigned i = 0; i != NumVecs; ++i)
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, RegVT, Value,
+ DAG.getVectorIdxConstant(i * RegElts, DL)));
+
+ Ops.push_back(PNg);
+ Ops.push_back(StoreNode->getBasePtr());
+
+ return DAG.getMemIntrinsicNode(
+ ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
+ StoreNode->getMemoryVT(), StoreNode->getMemOperand());
+}
+
// Custom lowering for any store, vector or scalar and/or default or with
// a truncate operations. Currently only custom lower truncate operation
// from vector v4i16 to v4i8 or volatile stores of i128.
@@ -7753,6 +7860,9 @@ SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
}
if (VT.isVector()) {
+ if (SDValue Store = tryLowerMultiVectorStore(StoreNode, DAG))
+ return Store;
+
if (useSVEForFixedLengthVectorVT(
VT,
/*OverrideNEON=*/Subtarget->useSVEForFixedLengthVectors()))
diff --git a/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll b/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
index 985326f26a2d5..e7a4ee355f58f 100644
--- a/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
+++ b/llvm/test/CodeGen/AArch64/sve-multivector-loads.ll
@@ -22,18 +22,18 @@ define void @load_2x_vectors_i8_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0]
; SVE2p1-SL-NEXT: add z1.b, z1.b, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.b, z0.b, #5 // =0x5
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1b { z0.b, z1.b }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i8_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
; SME2-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x0]
-; SME2-NEXT: add z24.b, z24.b, #5 // =0x5
-; SME2-NEXT: add z16.b, z16.b, #5 // =0x5
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.b, z1.b, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.b, z0.b, #5 // =0x5
+; SME2-NEXT: st1b { z0.b, z1.b }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 32 x i8>, ptr %addr
%b = add <vscale x 32 x i8> %a, splat (i8 5)
@@ -57,25 +57,23 @@ define void @load_2x_vectors_i8_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_2x_vectors_i8_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.b
-; SVE2p1-SL-NEXT: ptrue p0.b
; SVE2p1-SL-NEXT: add x8, x0, x1
; SVE2p1-SL-NEXT: ld1b { z0.b, z1.b }, pn8/z, [x0, x1]
-; SVE2p1-SL-NEXT: add z0.b, z0.b, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.b, z1.b, #5 // =0x5
-; SVE2p1-SL-NEXT: st1b { z0.b }, p0, [x0, x1]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.b, z0.b, #5 // =0x5
+; SVE2p1-SL-NEXT: st1b { z0.b, z1.b }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i8_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.b
-; SME2-NEXT: ptrue p0.b
; SME2-NEXT: add x8, x0, x1
; SME2-NEXT: ld1b { z16.b, z24.b }, pn8/z, [x0, x1]
-; SME2-NEXT: add z16.b, z16.b, #5 // =0x5
-; SME2-NEXT: add z24.b, z24.b, #5 // =0x5
-; SME2-NEXT: st1b { z16.b }, p0, [x0, x1]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.b, z1.b, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.b, z0.b, #5 // =0x5
+; SME2-NEXT: st1b { z0.b, z1.b }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i8, ptr %base, i64 %idx
%a = load <vscale x 32 x i8>, ptr %addr
@@ -101,18 +99,18 @@ define void @load_2x_vectors_i16_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0]
; SVE2p1-SL-NEXT: add z1.h, z1.h, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.h, z0.h, #5 // =0x5
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i16_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NEXT: add z24.h, z24.h, #5 // =0x5
-; SME2-NEXT: add z16.h, z16.h, #5 // =0x5
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.h, z1.h, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.h, z0.h, #5 // =0x5
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 16 x i16>, ptr %addr
%b = add <vscale x 16 x i16> %a, splat (i16 5)
@@ -136,25 +134,23 @@ define void @load_2x_vectors_i16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_2x_vectors_i16_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.h
-; SVE2p1-SL-NEXT: ptrue p0.h
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: add z0.h, z0.h, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.h, z1.h, #5 // =0x5
-; SVE2p1-SL-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.h, z0.h, #5 // =0x5
+; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i16_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.h
-; SME2-NEXT: ptrue p0.h
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: add z16.h, z16.h, #5 // =0x5
-; SME2-NEXT: add z24.h, z24.h, #5 // =0x5
-; SME2-NEXT: st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.h, z1.h, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.h, z0.h, #5 // =0x5
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i16, ptr %base, i64 %idx
%a = load <vscale x 16 x i16>, ptr %addr
@@ -180,18 +176,18 @@ define void @load_2x_vectors_i32_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0]
; SVE2p1-SL-NEXT: add z1.s, z1.s, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.s, z0.s, #5 // =0x5
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i32_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0]
-; SME2-NEXT: add z24.s, z24.s, #5 // =0x5
-; SME2-NEXT: add z16.s, z16.s, #5 // =0x5
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.s, z1.s, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.s, z0.s, #5 // =0x5
+; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 8 x i32>, ptr %addr
%b = add <vscale x 8 x i32> %a, splat (i32 5)
@@ -215,25 +211,23 @@ define void @load_2x_vectors_i32_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_2x_vectors_i32_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.s
-; SVE2p1-SL-NEXT: ptrue p0.s
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #2
; SVE2p1-SL-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: add z0.s, z0.s, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.s, z1.s, #5 // =0x5
-; SVE2p1-SL-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.s, z0.s, #5 // =0x5
+; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i32_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.s
-; SME2-NEXT: ptrue p0.s
; SME2-NEXT: add x8, x0, x1, lsl #2
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT: add z16.s, z16.s, #5 // =0x5
-; SME2-NEXT: add z24.s, z24.s, #5 // =0x5
-; SME2-NEXT: st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.s, z1.s, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.s, z0.s, #5 // =0x5
+; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i32, ptr %base, i64 %idx
%a = load <vscale x 8 x i32>, ptr %addr
@@ -259,18 +253,18 @@ define void @load_2x_vectors_i64_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0]
; SVE2p1-SL-NEXT: add z1.d, z1.d, #5 // =0x5
; SVE2p1-SL-NEXT: add z0.d, z0.d, #5 // =0x5
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i64_r:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0]
-; SME2-NEXT: add z24.d, z24.d, #5 // =0x5
-; SME2-NEXT: add z16.d, z16.d, #5 // =0x5
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.d, z1.d, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.d, z0.d, #5 // =0x5
+; SME2-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 4 x i64>, ptr %addr
%b = add <vscale x 4 x i64> %a, splat (i64 5)
@@ -294,25 +288,23 @@ define void @load_2x_vectors_i64_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-LABEL: load_2x_vectors_i64_rr:
; SVE2p1-SL: // %bb.0:
; SVE2p1-SL-NEXT: ptrue pn8.d
-; SVE2p1-SL-NEXT: ptrue p0.d
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #3
; SVE2p1-SL-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT: add z0.d, z0.d, #5 // =0x5
; SVE2p1-SL-NEXT: add z1.d, z1.d, #5 // =0x5
-; SVE2p1-SL-NEXT: st1d { z0.d }, p0, [x0, x1, lsl #3]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: add z0.d, z0.d, #5 // =0x5
+; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_i64_rr:
; SME2: // %bb.0:
; SME2-NEXT: ptrue pn8.d
-; SME2-NEXT: ptrue p0.d
; SME2-NEXT: add x8, x0, x1, lsl #3
; SME2-NEXT: ld1d { z16.d, z24.d }, pn8/z, [x0, x1, lsl #3]
-; SME2-NEXT: add z16.d, z16.d, #5 // =0x5
-; SME2-NEXT: add z24.d, z24.d, #5 // =0x5
-; SME2-NEXT: st1d { z16.d }, p0, [x0, x1, lsl #3]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: add z1.d, z1.d, #5 // =0x5
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: add z0.d, z0.d, #5 // =0x5
+; SME2-NEXT: st1d { z0.d, z1.d }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr i64, ptr %base, i64 %idx
%a = load <vscale x 4 x i64>, ptr %addr
@@ -340,8 +332,7 @@ define void @load_2x_vectors_f16_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0]
; SVE2p1-SL-NEXT: fadd z1.h, p0/m, z1.h, #1.0
; SVE2p1-SL-NEXT: fadd z0.h, p0/m, z0.h, #1.0
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f16_r:
@@ -349,10 +340,11 @@ define void @load_2x_vectors_f16_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.h
; SME2-NEXT: ptrue p0.h
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0]
-; SME2-NEXT: fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NEXT: fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: fadd z1.h, p0/m, z1.h, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.h, p0/m, z0.h, #1.0
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 16 x half>, ptr %addr
%b = fadd <vscale x 16 x half> %a, splat (half 1.0)
@@ -379,10 +371,9 @@ define void @load_2x_vectors_f16_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: ptrue p0.h
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #1
; SVE2p1-SL-NEXT: ld1h { z0.h, z1.h }, pn8/z, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: fadd z0.h, p0/m, z0.h, #1.0
; SVE2p1-SL-NEXT: fadd z1.h, p0/m, z1.h, #1.0
-; SVE2p1-SL-NEXT: st1h { z0.h }, p0, [x0, x1, lsl #1]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: fadd z0.h, p0/m, z0.h, #1.0
+; SVE2p1-SL-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f16_rr:
@@ -391,10 +382,11 @@ define void @load_2x_vectors_f16_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.h
; SME2-NEXT: add x8, x0, x1, lsl #1
; SME2-NEXT: ld1h { z16.h, z24.h }, pn8/z, [x0, x1, lsl #1]
-; SME2-NEXT: fadd z16.h, p0/m, z16.h, #1.0
-; SME2-NEXT: fadd z24.h, p0/m, z24.h, #1.0
-; SME2-NEXT: st1h { z16.h }, p0, [x0, x1, lsl #1]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: fadd z1.h, p0/m, z1.h, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.h, p0/m, z0.h, #1.0
+; SME2-NEXT: st1h { z0.h, z1.h }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr half, ptr %base, i64 %idx
%a = load <vscale x 16 x half>, ptr %addr
@@ -422,8 +414,7 @@ define void @load_2x_vectors_f32_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0]
; SVE2p1-SL-NEXT: fadd z1.s, p0/m, z1.s, #1.0
; SVE2p1-SL-NEXT: fadd z0.s, p0/m, z0.s, #1.0
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f32_r:
@@ -431,10 +422,11 @@ define void @load_2x_vectors_f32_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.s
; SME2-NEXT: ptrue p0.s
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0]
-; SME2-NEXT: fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NEXT: fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NEXT: str z24, [x0, #1, mul vl]
-; SME2-NEXT: str z16, [x0]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: fadd z1.s, p0/m, z1.s, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.s, p0/m, z0.s, #1.0
+; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x0]
; SME2-NEXT: ret
%a = load <vscale x 8 x float>, ptr %addr
%b = fadd <vscale x 8 x float> %a, splat (float 1.0)
@@ -461,10 +453,9 @@ define void @load_2x_vectors_f32_rr(ptr %base, i64 %idx) {
; SVE2p1-SL-NEXT: ptrue p0.s
; SVE2p1-SL-NEXT: add x8, x0, x1, lsl #2
; SVE2p1-SL-NEXT: ld1w { z0.s, z1.s }, pn8/z, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: fadd z0.s, p0/m, z0.s, #1.0
; SVE2p1-SL-NEXT: fadd z1.s, p0/m, z1.s, #1.0
-; SVE2p1-SL-NEXT: st1w { z0.s }, p0, [x0, x1, lsl #2]
-; SVE2p1-SL-NEXT: str z1, [x8, #1, mul vl]
+; SVE2p1-SL-NEXT: fadd z0.s, p0/m, z0.s, #1.0
+; SVE2p1-SL-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f32_rr:
@@ -473,10 +464,11 @@ define void @load_2x_vectors_f32_rr(ptr %base, i64 %idx) {
; SME2-NEXT: ptrue p0.s
; SME2-NEXT: add x8, x0, x1, lsl #2
; SME2-NEXT: ld1w { z16.s, z24.s }, pn8/z, [x0, x1, lsl #2]
-; SME2-NEXT: fadd z16.s, p0/m, z16.s, #1.0
-; SME2-NEXT: fadd z24.s, p0/m, z24.s, #1.0
-; SME2-NEXT: st1w { z16.s }, p0, [x0, x1, lsl #2]
-; SME2-NEXT: str z24, [x8, #1, mul vl]
+; SME2-NEXT: movprfx z1, z24
+; SME2-NEXT: fadd z1.s, p0/m, z1.s, #1.0
+; SME2-NEXT: movprfx z0, z16
+; SME2-NEXT: fadd z0.s, p0/m, z0.s, #1.0
+; SME2-NEXT: st1w { z0.s, z1.s }, pn8, [x8]
; SME2-NEXT: ret
%addr = getelementptr float, ptr %base, i64 %idx
%a = load <vscale x 8 x float>, ptr %addr
@@ -504,8 +496,7 @@ define void @load_2x_vectors_f64_r(ptr %addr) {
; SVE2p1-SL-NEXT: ld1d { z0.d, z1.d }, pn8/z, [x0]
; SVE2p1-SL-NEXT: fadd z1.d, p0/m, z1.d, #1.0
; SVE2p1-SL-NEXT: fadd z0.d, p0/m, z0.d, #1.0
-; SVE2p1-SL-NEXT: str z1, [x0, #1, mul vl]
-; SVE2p1-SL-NEXT: str z0, [x0]
+; SVE2p1-SL-NEXT: st1d { z0.d, z1.d }, pn8, [x0]
; SVE2p1-SL-NEXT: ret
;
; SME2-LABEL: load_2x_vectors_f64_r:
@@ -513,10 +504,11 @@ define void @load_2x_vectors_f64_r(ptr %addr) {
; SME2-NEXT: ptrue pn8.d
; SME2-NEXT: ptrue p0.d
; SME2-NEXT...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/207397
More information about the llvm-commits
mailing list