[llvm] [AArch64] Preserve store memoperands (PR #210936)
Kamlesh Kumar via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 21 03:44:59 PDT 2026
https://github.com/kamleshbhalui created https://github.com/llvm/llvm-project/pull/210936
preserve memoperands for structured store.
>From 55e38d953d5573d7c01afd71399c89edc00826cf Mon Sep 17 00:00:00 2001
From: Kamlesh Kumar <kamlesh.kumar at arm.com>
Date: Tue, 21 Jul 2026 05:24:19 +0000
Subject: [PATCH] [AArch64] Preserve store memoperands
---
.../Target/AArch64/AArch64ISelDAGToDAG.cpp | 4 +
.../Target/AArch64/AArch64ISelLowering.cpp | 17 +++-
.../sve-structured-store-memoperands.ll | 77 +++++++++++++++++++
.../CodeGen/AArch64/sve-vector-interleave.ll | 2 +-
4 files changed, 96 insertions(+), 4 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/sve-structured-store-memoperands.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 3f23e34d54c55..c3f1650ca0d6f 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -2564,6 +2564,10 @@ void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
N->getOperand(0)}; // chain
SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops);
+ // Transfer memoperands.
+ MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
+ CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
+
ReplaceNode(N, St);
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 613fdd2813f63..2422e1cec1341 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -18358,14 +18358,17 @@ void AArch64TargetLowering::getTgtMemIntrinsic(
auto &DL = I.getDataLayout();
switch (Intrinsic) {
case Intrinsic::aarch64_sve_st2:
+ case Intrinsic::aarch64_sve_st2q:
setInfoSVEStN<2>(*this, DL, Info, I);
Infos.push_back(Info);
return;
case Intrinsic::aarch64_sve_st3:
+ case Intrinsic::aarch64_sve_st3q:
setInfoSVEStN<3>(*this, DL, Info, I);
Infos.push_back(Info);
return;
case Intrinsic::aarch64_sve_st4:
+ case Intrinsic::aarch64_sve_st4q:
setInfoSVEStN<4>(*this, DL, Info, I);
Infos.push_back(Info);
return;
@@ -34457,7 +34460,8 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
Align Alignment = DAG.getReducedAlign(OpVT, /*UseABI=*/false);
SDValue StackPtr =
DAG.CreateStackTemporary(OpVT.getStoreSize() * 3, Alignment);
-
+ MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack(
+ DAG.getMachineFunction(), cast<FrameIndexSDNode>(StackPtr)->getIndex());
SmallVector<SDValue, 6> Ops;
Ops.push_back(DAG.getEntryNode());
Ops.push_back(
@@ -34471,7 +34475,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
OpVT.getVectorNumElements() * 3);
SDValue Chain = DAG.getMemIntrinsicNode(
ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, TripleOpVT,
- MachinePointerInfo(), Alignment, MachineMemOperand::MOStore);
+ StackPtrInfo, Alignment, MachineMemOperand::MOStore);
SmallVector<SDValue, 3> Results;
for (unsigned I = 0; I < 3; ++I) {
@@ -34493,6 +34497,8 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
Align Alignment = DAG.getReducedAlign(PackedVT, /*UseABI=*/false);
SDValue StackPtr =
DAG.CreateStackTemporary(PackedVT.getStoreSize() * 3, Alignment);
+ MachinePointerInfo StackPtrInfo = MachinePointerInfo::getFixedStack(
+ DAG.getMachineFunction(), cast<FrameIndexSDNode>(StackPtr)->getIndex());
Intrinsic::ID IntID = Intrinsic::aarch64_sve_st3;
EVT PredVT = PackedVT.changeVectorElementType(*DAG.getContext(), MVT::i1);
@@ -34505,7 +34511,12 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
Ops.push_back(StackPtr);
// Interleave operands and store.
- SDValue Chain = DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops);
+ EVT TriplePackedVT =
+ EVT::getVectorVT(*DAG.getContext(), PackedVT.getVectorElementType(),
+ PackedVT.getVectorElementCount() * 3);
+ SDValue Chain = DAG.getMemIntrinsicNode(
+ ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, TriplePackedVT,
+ StackPtrInfo, Alignment, MachineMemOperand::MOStore);
// Read back the interleaved data.
SmallVector<SDValue, 3> Results;
diff --git a/llvm/test/CodeGen/AArch64/sve-structured-store-memoperands.ll b/llvm/test/CodeGen/AArch64/sve-structured-store-memoperands.ll
new file mode 100644
index 0000000000000..efe79cce11e2e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-structured-store-memoperands.ll
@@ -0,0 +1,77 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve -stop-after=aarch64-isel -verify-machineinstrs -o - < %s | FileCheck %s
+
+define void @st2b_imm_mmo(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1,
+ <vscale x 16 x i1> %pred, ptr %addr) {
+; CHECK-LABEL: name: st2b_imm_mmo
+; CHECK: ST2B_IMM {{.*}} :: (store (<vscale x 1 x s256>) into %ir.base)
+ %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 2, i64 0
+ call void @llvm.aarch64.sve.st2.nxv16i8(<vscale x 16 x i8> %v0,
+ <vscale x 16 x i8> %v1,
+ <vscale x 16 x i1> %pred,
+ ptr %base)
+ ret void
+}
+
+define void @st2b_reg_mmo(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1,
+ <vscale x 16 x i1> %pred, ptr %addr,
+ i64 %offset) {
+; CHECK-LABEL: name: st2b_reg_mmo
+; CHECK: ST2B {{.*}} :: (store (<vscale x 1 x s256>) into %ir.base)
+ %base = getelementptr <vscale x 16 x i8>, ptr %addr, i64 %offset, i64 0
+ call void @llvm.aarch64.sve.st2.nxv16i8(<vscale x 16 x i8> %v0,
+ <vscale x 16 x i8> %v1,
+ <vscale x 16 x i1> %pred,
+ ptr %base)
+ ret void
+}
+
+define void @st2q_mmo(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1,
+ <vscale x 16 x i1> %pred, ptr %addr) {
+; CHECK-LABEL: name: st2q_mmo
+; CHECK: ST2Q_IMM {{.*}} :: (store (<vscale x 1 x s256>) into %ir.addr)
+ call void @llvm.aarch64.sve.st2q.nxv16i8(<vscale x 16 x i8> %v0,
+ <vscale x 16 x i8> %v1,
+ <vscale x 16 x i1> %pred,
+ ptr %addr)
+ ret void
+}
+
+define void @st3q_mmo(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1,
+ <vscale x 16 x i8> %v2,
+ <vscale x 16 x i1> %pred, ptr %addr) {
+; CHECK-LABEL: name: st3q_mmo
+; CHECK: ST3Q_IMM {{.*}} :: (store (<vscale x 1 x s384>) into %ir.addr
+ call void @llvm.aarch64.sve.st3q.nxv16i8(<vscale x 16 x i8> %v0,
+ <vscale x 16 x i8> %v1,
+ <vscale x 16 x i8> %v2,
+ <vscale x 16 x i1> %pred,
+ ptr %addr)
+ ret void
+}
+
+define void @st4q_mmo(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1,
+ <vscale x 16 x i8> %v2, <vscale x 16 x i8> %v3,
+ <vscale x 16 x i1> %pred, ptr %addr) {
+; CHECK-LABEL: name: st4q_mmo
+; CHECK: ST4Q_IMM {{.*}} :: (store (<vscale x 1 x s512>) into %ir.addr
+ call void @llvm.aarch64.sve.st4q.nxv16i8(<vscale x 16 x i8> %v0,
+ <vscale x 16 x i8> %v1,
+ <vscale x 16 x i8> %v2,
+ <vscale x 16 x i8> %v3,
+ <vscale x 16 x i1> %pred,
+ ptr %addr)
+ ret void
+}
+
+define <vscale x 6 x half>
+ at interleave3_nxv6f16_mmo(<vscale x 2 x half> %v0,
+ <vscale x 2 x half> %v1,
+ <vscale x 2 x half> %v2) {
+; CHECK-LABEL: name: interleave3_nxv6f16_mmo
+; CHECK: ST3D_IMM {{.*}} :: (store (<vscale x 1 x s384>) into %stack.0, align 16)
+ %interleaved = call <vscale x 6 x half>
+ @llvm.vector.interleave3.nxv6f16(<vscale x 2 x half> %v0,
+ <vscale x 2 x half> %v1,
+ <vscale x 2 x half> %v2)
+ ret <vscale x 6 x half> %interleaved
+}
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
index 81a0cc242fc11..c497ba6f7008b 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
@@ -876,8 +876,8 @@ define <vscale x 12 x i64> @interleave6_nxv12i64(<vscale x 2 x i64> %vec0, <vsca
; SME2-NEXT: mov z3.d, z7.d
; SME2-NEXT: ptrue p0.d
; SME2-NEXT: st3d { z4.d - z6.d }, p0, [sp]
-; SME2-NEXT: st3d { z1.d - z3.d }, p0, [sp, #3, mul vl]
; SME2-NEXT: ldr z0, [sp]
+; SME2-NEXT: st3d { z1.d - z3.d }, p0, [sp, #3, mul vl]
; SME2-NEXT: ldr z1, [sp, #1, mul vl]
; SME2-NEXT: ldr z2, [sp, #2, mul vl]
; SME2-NEXT: ldr z3, [sp, #3, mul vl]
More information about the llvm-commits
mailing list