[llvm] 13bdd50 - [AArch64][ISel] Enable masked interleaved stores for splat values (#207950)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 01:50:17 PDT 2026
Author: Harry Ramsey
Date: 2026-07-15T09:50:12+01:00
New Revision: 13bdd50540a5365f5f25b0fff7410d5d69157a51
URL: https://github.com/llvm/llvm-project/commit/13bdd50540a5365f5f25b0fff7410d5d69157a51
DIFF: https://github.com/llvm/llvm-project/commit/13bdd50540a5365f5f25b0fff7410d5d69157a51.diff
LOG: [AArch64][ISel] Enable masked interleaved stores for splat values (#207950)
Enable masked interleaved store combine to recognise splat values and
split them into SVE component vectors. This allows the existing stN
lowering path to handle masked stores where the stored value is a splat
rather than an explicit vector.interleave result.
Added:
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index b01dfe2f1b925..85342c387fa3e 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27531,6 +27531,43 @@ isSequentialConcatOfVectorInterleave(SDNode *N, SmallVectorImpl<SDValue> &Ops) {
return true;
}
+static bool isSplatVectorInterleaveOps(SelectionDAG &DAG, SDLoc DL,
+ SDValue WideValue,
+ SmallVectorImpl<SDValue> &Ops) {
+ EVT WideVT = WideValue.getValueType();
+ if (!WideVT.isScalableVector())
+ return false;
+
+ SDValue SplatValue = DAG.getSplatValue(WideValue);
+ if (!SplatValue)
+ return false;
+
+ TypeSize WideSize = WideVT.getSizeInBits();
+ if (!WideSize.isKnownMultipleOf(128))
+ return false;
+
+ unsigned NumParts = WideSize.getKnownMinValue() / 128;
+ if (NumParts != 2 && NumParts != 3 && NumParts != 4)
+ return false;
+
+ ElementCount WideEC = WideVT.getVectorElementCount();
+ if (!WideEC.isKnownMultipleOf(NumParts))
+ return false;
+
+ EVT SubVecTy =
+ EVT::getVectorVT(*DAG.getContext(), WideVT.getVectorElementType(),
+ WideEC.divideCoefficientBy(NumParts));
+
+ if (SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
+ !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+ return false;
+
+ SDValue NarrowSplat =
+ DAG.getNode(ISD::SPLAT_VECTOR, DL, SubVecTy, SplatValue);
+ Ops.append(NumParts, NarrowSplat);
+ return true;
+}
+
static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL,
SDValue WideMask,
unsigned RequiredNumParts) {
@@ -27569,17 +27606,24 @@ static SDValue performInterleavedMaskedStoreCombine(
MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
SDValue WideValue = MST->getValue();
- // Bail out if the stored value has an unexpected number of uses, since we'll
- // have to perform manual interleaving and may as well just use normal masked
- // stores. Also, discard masked stores that are truncating or indexed.
- if (!WideValue.hasOneUse() || !ISD::isNormalMaskedStore(MST) ||
- !MST->isSimple() || !MST->getOffset().isUndef())
+ // Discard masked stores that are truncating or indexed.
+ if (!ISD::isNormalMaskedStore(MST) || !MST->isSimple() ||
+ !MST->getOffset().isUndef())
return SDValue();
+ SDLoc DL(N);
SmallVector<SDValue, 4> ValueInterleaveOps;
- if (!isSequentialConcatOfVectorInterleave(WideValue.getNode(),
- ValueInterleaveOps))
+ if (isSequentialConcatOfVectorInterleave(WideValue.getNode(),
+ ValueInterleaveOps)) {
+ // Bail out if the stored value has an unexpected number of uses, since
+ // we'll have to perform manual interleaving and may as well just use normal
+ // masked stores.
+ if (!WideValue.hasOneUse())
+ return SDValue();
+ } else if (!isSplatVectorInterleaveOps(DAG, DL, WideValue,
+ ValueInterleaveOps)) {
return SDValue();
+ }
unsigned NumParts = ValueInterleaveOps.size();
if (NumParts != 2 && NumParts != 4)
@@ -27593,7 +27637,6 @@ static SDValue performInterleavedMaskedStoreCombine(
!DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
return SDValue();
- SDLoc DL(N);
SDValue NarrowMask =
getNarrowMaskForInterleavedOps(DAG, DL, MST->getMask(), NumParts);
if (!NarrowMask)
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index d3cd9bf08cc0a..87833306571c2 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -282,3 +282,59 @@ define void @foo_st2_nxv8i8_trunc(<vscale x 4 x i1> %mask, <vscale x 4 x i16> %v
call void @llvm.masked.store.nxv8i8.p0(<vscale x 8 x i8> %trunc.value, ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask)
ret void
}
+
+define void @foo_st2_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv16i8_zeroinitializer:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> zeroinitializer, ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st4_nxv16i8_zeroinitializer(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv16i8_zeroinitializer:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4b { z0.b - z3.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> zeroinitializer, ptr %p, i32 1, <vscale x 64 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st2_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv16i8_splat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.b, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %base = insertelement <vscale x 32 x i8> poison, i8 1, i32 0
+ %interleaved.value = shufflevector <vscale x 32 x i8> %base, <vscale x 32 x i8> poison, <vscale x 32 x i32> zeroinitializer
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv32i8.p0(<vscale x 32 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st4_nxv16i8_splat(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv16i8_splat:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.b, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4b { z0.b - z3.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %base = insertelement <vscale x 64 x i8> poison, i8 1, i32 0
+ %interleaved.value = shufflevector <vscale x 64 x i8> %base, <vscale x 64 x i8> poison, <vscale x 64 x i32> zeroinitializer
+ %interleaved.mask = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv64i8.p0(<vscale x 64 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 64 x i1> %interleaved.mask)
+ ret void
+}
More information about the llvm-commits
mailing list