[llvm] [AArch64][SDAG] Enable +sme2p2/+sve2p2 lowering for compressstore (PR #218624)
Jack Styles via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 27 02:07:43 PDT 2026
https://github.com/Stylie777 updated https://github.com/llvm/llvm-project/pull/218624
>From cb617e2f9f6a33d7138787d374a2da7f0bffe9ba Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 24 Aug 2026 09:44:28 +0100
Subject: [PATCH 1/9] [AArch64][SDAG][NFC] Use CNTP intrinsic for MSTORE
lowering
Currently, when lowering MSTORE the process is to Zero Extend,
VECREDUCE_ADD and then Zero Extend the result for types that are
not i64. This for i8/i16 types does not work as AArch64 cannot
zero extend these types to i64. Instead, use the CNTP intrinsic
to combine the values before storing them.
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 36a86241fe3b2..c1ca0f789e534 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33922,16 +33922,11 @@ SDValue AArch64TargetLowering::LowerMSTORE(SDValue Op,
return SDValue();
EVT MaskVT = Store->getMask().getValueType();
- EVT MaskExtVT = getPromotedVTForPredicate(MaskVT);
- EVT MaskReduceVT = MaskExtVT.getScalarType();
SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
-
- SDValue MaskExt =
- DAG.getNode(ISD::ZERO_EXTEND, DL, MaskExtVT, Store->getMask());
- SDValue CntActive =
- DAG.getNode(ISD::VECREDUCE_ADD, DL, MaskReduceVT, MaskExt);
- if (MaskReduceVT != MVT::i64)
- CntActive = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, CntActive);
+ SDValue CntActive = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64,
+ DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64),
+ Store->getMask(), Store->getMask());
SDValue CompressedValue =
DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, Store->getValue(),
>From 785b0aa4f2e22c887458160186ad84bcae3befa0 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 20 Aug 2026 13:52:13 +0100
Subject: [PATCH 2/9] [AArch64][SDAG] Enable +sme2p2/+sve2p2 lowering for
MSTORE
Following on from #215219 it will now be possible to lower
llvm.masked.compressstore for sve2p2/sme2p2 using compact
for i8/i16/f16/bf16 types.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 6 +-
.../AArch64/AArch64TargetTransformInfo.h | 5 ++
.../CostModel/AArch64/masked_compress_load.ll | 74 +++++++++----------
.../sve-masked-compressstore-sve2p2.ll | 48 ++++++++++--
4 files changed, 88 insertions(+), 45 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c1ca0f789e534..6a08bf6bc0e38 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2283,6 +2283,11 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
// lowering for non-compressing masked stores.
setOperationAction(ISD::MSTORE, VT, Custom);
}
+ if (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()) {
+ // With +sve2p2/+sme2p2 the full range of vector types are supported.
+ for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16})
+ setOperationAction(ISD::MSTORE, VT, Custom);
+ }
// Histcnt is SVE2 only
if (Subtarget->hasSVE2()) {
@@ -33927,7 +33932,6 @@ SDValue AArch64TargetLowering::LowerMSTORE(SDValue Op,
ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64,
DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64),
Store->getMask(), Store->getMask());
-
SDValue CompressedValue =
DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, Store->getValue(),
Store->getMask(), DAG.getPOISON(VT));
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 19de90427cb74..64337ff4d5db7 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -334,6 +334,11 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
}
bool isElementTypeLegalForCompressStore(Type *Ty) const {
+ if ((ST->hasSVE2p2() || ST->hasSME2p2()) &&
+ ((Ty->isIntegerTy(8) || Ty->isIntegerTy(16)) ||
+ ((Ty->isHalfTy() || Ty->isBFloatTy()) &&
+ Ty->getScalarSizeInBits() == 16)))
+ return true;
return Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isIntegerTy(32) ||
Ty->isIntegerTy(64);
}
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
index cb59f1016a4d1..7cc880ce5b66d 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
@@ -95,21 +95,21 @@ define void @fixed() {
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i8.p0(<2 x i8> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i8.p0(<4 x i8> poison, ptr poison, <4 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i8.p0(<8 x i8> poison, ptr poison, <8 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i16.p0(<2 x i16> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i16.p0(<4 x i16> poison, ptr poison, <4 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i32.p0(<2 x i32> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f16.p0(<2 x half> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4f16.p0(<4 x half> poison, ptr poison, <4 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f32.p0(<2 x float> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
entry:
@@ -142,25 +142,25 @@ entry:
define void @scalable() {
; SVE2p2-SME2p2-NON-STREAMING-LABEL: 'scalable'
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE2p2-SME2p2-STREAMING-LABEL: 'scalable'
@@ -186,25 +186,25 @@ define void @scalable() {
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE2p2-SME2p2-LABEL: 'scalable'
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
; SVE2p2-SME2p2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE-ONLY-LABEL: 'scalable'
@@ -230,25 +230,25 @@ define void @scalable() {
; SVE-ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE2p2-SME2p2-SVE256-LABEL: 'scalable'
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
; SVE2p2-SME2p2-SVE256-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
entry:
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll b/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll
index 92ecc3c83e2c5..5655d53b5ddaa 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll
@@ -1,17 +1,51 @@
-; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 < %s
-
-;; These masked.compressstore operations could be natively supported with +sve2p2
-;; (or by promoting to 32/64 bit elements + a truncstore), but currently are not
-;; supported.
-
-; XFAIL: *
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE --allow-unused-prefixes
+; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=256 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VL256 --allow-unused-prefixes
define void @test_compressstore_nxv8i16(ptr %p, <vscale x 8 x i16> %vec, <vscale x 8 x i1> %mask) {
+; CHECK-LABEL: test_compressstore_nxv8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cntp x8, p0, p0.h
+; CHECK-NEXT: compact z0.h, p0, z0.h
+; CHECK-NEXT: whilelo p1.h, xzr, x8
+; CHECK-NEXT: st1h { z0.h }, p1, [x0]
+; CHECK-NEXT: ret
tail call void @llvm.masked.compressstore.nxv8i16(<vscale x 8 x i16> %vec, ptr align 2 %p, <vscale x 8 x i1> %mask)
ret void
}
define void @test_compressstore_nxv16i8(ptr %p, <vscale x 16 x i8> %vec, <vscale x 16 x i1> %mask) {
+; CHECK-LABEL: test_compressstore_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cntp x8, p0, p0.b
+; CHECK-NEXT: compact z0.b, p0, z0.b
+; CHECK-NEXT: whilelo p1.b, xzr, x8
+; CHECK-NEXT: st1b { z0.b }, p1, [x0]
+; CHECK-NEXT: ret
tail call void @llvm.masked.compressstore.nxv16i8(<vscale x 16 x i8> %vec, ptr align 1 %p, <vscale x 16 x i1> %mask)
ret void
}
+
+define void @test_compressstore_nxv8f16(ptr %p, <vscale x 8 x half> %vec, <vscale x 8 x i1> %mask) {
+; CHECK-LABEL: test_compressstore_nxv8f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cntp x8, p0, p0.h
+; CHECK-NEXT: compact z0.h, p0, z0.h
+; CHECK-NEXT: whilelo p1.h, xzr, x8
+; CHECK-NEXT: st1h { z0.h }, p1, [x0]
+; CHECK-NEXT: ret
+ tail call void @llvm.masked.compressstore.nxv8f16(<vscale x 8 x half> %vec, ptr align 1 %p, <vscale x 8 x i1> %mask)
+ ret void
+}
+
+define void @test_compressstore_nxv8bf16(ptr %p, <vscale x 8 x bfloat> %vec, <vscale x 8 x i1> %mask) {
+; CHECK-LABEL: test_compressstore_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cntp x8, p0, p0.h
+; CHECK-NEXT: compact z0.h, p0, z0.h
+; CHECK-NEXT: whilelo p1.h, xzr, x8
+; CHECK-NEXT: st1h { z0.h }, p1, [x0]
+; CHECK-NEXT: ret
+ tail call void @llvm.masked.compressstore.nxv8bf16(<vscale x 8 x bfloat> %vec, ptr align 1 %p, <vscale x 8 x i1> %mask)
+ ret void
+}
>From f8a722e61b99837d18fda72e3460662e8d5f8292 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 24 Aug 2026 10:03:30 +0100
Subject: [PATCH 3/9] Remove whitespace change
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 6a08bf6bc0e38..07b28395c0dfe 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -33932,6 +33932,7 @@ SDValue AArch64TargetLowering::LowerMSTORE(SDValue Op,
ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64,
DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64),
Store->getMask(), Store->getMask());
+
SDValue CompressedValue =
DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, Store->getValue(),
Store->getMask(), DAG.getPOISON(VT));
>From 8e09f73faf8315ede14485f2797935482a9a3452 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 24 Aug 2026 10:48:06 +0100
Subject: [PATCH 4/9] Enable Streaming SME2p2
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 11 ++++++-----
.../AArch64/sve-masked-compressstore-sve2p2.ll | 3 +++
2 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 07b28395c0dfe..639d5b079f734 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2283,11 +2283,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
// lowering for non-compressing masked stores.
setOperationAction(ISD::MSTORE, VT, Custom);
}
- if (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()) {
- // With +sve2p2/+sme2p2 the full range of vector types are supported.
- for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16})
- setOperationAction(ISD::MSTORE, VT, Custom);
- }
// Histcnt is SVE2 only
if (Subtarget->hasSVE2()) {
@@ -2308,6 +2303,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
}
}
+ if ((Subtarget->isSVEAvailable() && Subtarget->hasSVE2p2()) || (Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())) {
+ // With +sve2p2/+sme2p2 the full range of vector types are supported.
+ for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16})
+ setOperationAction(ISD::MSTORE, VT, Custom);
+ }
+
if (Subtarget->hasMOPS() && Subtarget->hasMTE()) {
// Only required for llvm.aarch64.mops.memset.tag
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll b/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll
index 5655d53b5ddaa..4fca173a712ba 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll
@@ -1,6 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE --allow-unused-prefixes
; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=256 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VL256 --allow-unused-prefixes
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2p2 --allow-unused-prefixes
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p2 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SME2p2-STREAMING --allow-unused-prefixes
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p2,+sve2p2 -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-STREAMING-COMPAT --allow-unused-prefixes
define void @test_compressstore_nxv8i16(ptr %p, <vscale x 8 x i16> %vec, <vscale x 8 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv8i16:
>From 3865d30d79af0986c28e23b2f713e22d70ebd905 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Mon, 24 Aug 2026 10:50:40 +0100
Subject: [PATCH 5/9] format
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 639d5b079f734..a0fb5365cd843 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2303,7 +2303,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
}
}
- if ((Subtarget->isSVEAvailable() && Subtarget->hasSVE2p2()) || (Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())) {
+ if ((Subtarget->isSVEAvailable() && Subtarget->hasSVE2p2()) ||
+ (Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())) {
// With +sve2p2/+sme2p2 the full range of vector types are supported.
for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16})
setOperationAction(ISD::MSTORE, VT, Custom);
>From 929e6c0139c46fea7227a2185de5a0ef4987ba73 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 26 Aug 2026 14:38:12 +0100
Subject: [PATCH 6/9] Respond to review comments
---
.../Target/AArch64/AArch64ISelLowering.cpp | 11 +-
.../AArch64/AArch64TargetTransformInfo.h | 8 +-
.../sve-masked-compressstore-sve2p2.ll | 289 +++++++++++++++++-
3 files changed, 290 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 555a3582c37ee..cdbf7fed84e10 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2227,8 +2227,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
if (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()) {
// With +sve2p2/+sme2p2 the full range of vector types are supported.
- for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16})
+ for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16}) {
+ setOperationAction(ISD::MSTORE, VT, Custom);
setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
+ }
for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v4f16,
MVT::v8f16, MVT::v4bf16, MVT::v8bf16})
@@ -2303,13 +2305,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
}
}
- if ((Subtarget->isSVEAvailable() && Subtarget->hasSVE2p2()) ||
- (Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())) {
- // With +sve2p2/+sme2p2 the full range of vector types are supported.
- for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16})
- setOperationAction(ISD::MSTORE, VT, Custom);
- }
-
if (Subtarget->hasMOPS() && Subtarget->hasMTE()) {
// Only required for llvm.aarch64.mops.memset.tag
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index a6cb5bdebcfab..5c7b2703b4257 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -334,10 +334,8 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
}
bool isElementTypeLegalForCompressStore(Type *Ty) const {
- if ((ST->hasSVE2p2() || ST->hasSME2p2()) &&
- ((Ty->isIntegerTy(8) || Ty->isIntegerTy(16)) ||
- ((Ty->isHalfTy() || Ty->isBFloatTy()) &&
- Ty->getScalarSizeInBits() == 16)))
+ if ((ST->hasSVE2p2() || (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2())) &&
+ (Ty->isIntegerTy(8) || Ty->isIntegerTy(16) || Ty->getScalarSizeInBits() == 16))
return true;
return Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isIntegerTy(32) ||
Ty->isIntegerTy(64);
@@ -345,7 +343,7 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
bool isLegalMaskedCompressStore(Type *DataType,
Align Alignment) const override {
- if (!ST->isSVEAvailable())
+ if (!ST->isSVEorStreamingSVEAvailable())
return false;
if (isa<FixedVectorType>(DataType) &&
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll b/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll
index 4fca173a712ba..42cef84285a0f 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-compressstore-sve2p2.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE --allow-unused-prefixes
-; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=256 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VL256 --allow-unused-prefixes
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2p2 --allow-unused-prefixes
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p2 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SME2p2-STREAMING --allow-unused-prefixes
-; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p2,+sve2p2 -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-STREAMING-COMPAT --allow-unused-prefixes
+; RUN: llc -mtriple=aarch64 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-BASE
+; RUN: llc -mtriple=aarch64 -aarch64-sve-vector-bits-min=256 -mattr=+sve2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-VL256
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme2p2 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE2p2
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p2 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SME2p2-STREAMING
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2p2,+sve2p2 -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-STREAMING-COMPAT
define void @test_compressstore_nxv8i16(ptr %p, <vscale x 8 x i16> %vec, <vscale x 8 x i1> %mask) {
; CHECK-LABEL: test_compressstore_nxv8i16:
@@ -52,3 +52,282 @@ define void @test_compressstore_nxv8bf16(ptr %p, <vscale x 8 x bfloat> %vec, <vs
tail call void @llvm.masked.compressstore.nxv8bf16(<vscale x 8 x bfloat> %vec, ptr align 1 %p, <vscale x 8 x i1> %mask)
ret void
}
+
+define void @test_compressstore_v8i16(ptr %p, <8 x i16> %vec, <8 x i1> %mask) {
+; CHECK-BASE-LABEL: test_compressstore_v8i16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-BASE-NEXT: ptrue p0.h, vl8
+; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-BASE-NEXT: shl v1.8h, v1.8h, #15
+; CHECK-BASE-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-BASE-NEXT: cntp x8, p1, p1.h
+; CHECK-BASE-NEXT: compact z0.h, p1, z0.h
+; CHECK-BASE-NEXT: whilelo p0.h, xzr, x8
+; CHECK-BASE-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-VL256-LABEL: test_compressstore_v8i16:
+; CHECK-VL256: // %bb.0:
+; CHECK-VL256-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-VL256-NEXT: ptrue p0.h, vl8
+; CHECK-VL256-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-VL256-NEXT: shl v1.8h, v1.8h, #15
+; CHECK-VL256-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-VL256-NEXT: cntp x8, p1, p1.h
+; CHECK-VL256-NEXT: compact z0.h, p1, z0.h
+; CHECK-VL256-NEXT: whilelo p0.h, xzr, x8
+; CHECK-VL256-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-VL256-NEXT: ret
+;
+; CHECK-SVE2p2-LABEL: test_compressstore_v8i16:
+; CHECK-SVE2p2: // %bb.0:
+; CHECK-SVE2p2-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-SVE2p2-NEXT: ptrue p0.h, vl8
+; CHECK-SVE2p2-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-SVE2p2-NEXT: shl v1.8h, v1.8h, #15
+; CHECK-SVE2p2-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SVE2p2-NEXT: cntp x8, p1, p1.h
+; CHECK-SVE2p2-NEXT: compact z0.h, p1, z0.h
+; CHECK-SVE2p2-NEXT: whilelo p0.h, xzr, x8
+; CHECK-SVE2p2-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-SVE2p2-NEXT: ret
+;
+; CHECK-SME2p2-STREAMING-LABEL: test_compressstore_v8i16:
+; CHECK-SME2p2-STREAMING: // %bb.0:
+; CHECK-SME2p2-STREAMING-NEXT: uunpklo z1.h, z1.b
+; CHECK-SME2p2-STREAMING-NEXT: ptrue p0.h, vl8
+; CHECK-SME2p2-STREAMING-NEXT: lsl z1.h, z1.h, #15
+; CHECK-SME2p2-STREAMING-NEXT: asr z1.h, z1.h, #15
+; CHECK-SME2p2-STREAMING-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SME2p2-STREAMING-NEXT: cntp x8, p1, p1.h
+; CHECK-SME2p2-STREAMING-NEXT: compact z0.h, p1, z0.h
+; CHECK-SME2p2-STREAMING-NEXT: whilelo p0.h, xzr, x8
+; CHECK-SME2p2-STREAMING-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-SME2p2-STREAMING-NEXT: ret
+;
+; CHECK-STREAMING-COMPAT-LABEL: test_compressstore_v8i16:
+; CHECK-STREAMING-COMPAT: // %bb.0:
+; CHECK-STREAMING-COMPAT-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-STREAMING-COMPAT-NEXT: ptrue p0.h, vl8
+; CHECK-STREAMING-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-STREAMING-COMPAT-NEXT: uunpklo z1.h, z1.b
+; CHECK-STREAMING-COMPAT-NEXT: lsl z1.h, z1.h, #15
+; CHECK-STREAMING-COMPAT-NEXT: asr z1.h, z1.h, #15
+; CHECK-STREAMING-COMPAT-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-STREAMING-COMPAT-NEXT: cntp x8, p1, p1.h
+; CHECK-STREAMING-COMPAT-NEXT: compact z0.h, p1, z0.h
+; CHECK-STREAMING-COMPAT-NEXT: whilelo p0.h, xzr, x8
+; CHECK-STREAMING-COMPAT-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-STREAMING-COMPAT-NEXT: ret
+ tail call void @llvm.masked.compressstore.v8i16(<8 x i16> %vec, ptr align 2 %p, <8 x i1> %mask)
+ ret void
+}
+
+define void @test_compressstore_v16i8(ptr %p, <16 x i8> %vec, <16 x i1> %mask) {
+; CHECK-BASE-LABEL: test_compressstore_v16i8:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: shl v1.16b, v1.16b, #7
+; CHECK-BASE-NEXT: ptrue p0.b, vl16
+; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-BASE-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; CHECK-BASE-NEXT: cntp x8, p1, p1.b
+; CHECK-BASE-NEXT: compact z0.b, p1, z0.b
+; CHECK-BASE-NEXT: whilelo p0.b, xzr, x8
+; CHECK-BASE-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-VL256-LABEL: test_compressstore_v16i8:
+; CHECK-VL256: // %bb.0:
+; CHECK-VL256-NEXT: shl v1.16b, v1.16b, #7
+; CHECK-VL256-NEXT: ptrue p0.b, vl16
+; CHECK-VL256-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-VL256-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; CHECK-VL256-NEXT: cntp x8, p1, p1.b
+; CHECK-VL256-NEXT: compact z0.b, p1, z0.b
+; CHECK-VL256-NEXT: whilelo p0.b, xzr, x8
+; CHECK-VL256-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-VL256-NEXT: ret
+;
+; CHECK-SVE2p2-LABEL: test_compressstore_v16i8:
+; CHECK-SVE2p2: // %bb.0:
+; CHECK-SVE2p2-NEXT: shl v1.16b, v1.16b, #7
+; CHECK-SVE2p2-NEXT: ptrue p0.b, vl16
+; CHECK-SVE2p2-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-SVE2p2-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; CHECK-SVE2p2-NEXT: cntp x8, p1, p1.b
+; CHECK-SVE2p2-NEXT: compact z0.b, p1, z0.b
+; CHECK-SVE2p2-NEXT: whilelo p0.b, xzr, x8
+; CHECK-SVE2p2-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-SVE2p2-NEXT: ret
+;
+; CHECK-SME2p2-STREAMING-LABEL: test_compressstore_v16i8:
+; CHECK-SME2p2-STREAMING: // %bb.0:
+; CHECK-SME2p2-STREAMING-NEXT: lsl z1.b, z1.b, #7
+; CHECK-SME2p2-STREAMING-NEXT: ptrue p0.b, vl16
+; CHECK-SME2p2-STREAMING-NEXT: asr z1.b, z1.b, #7
+; CHECK-SME2p2-STREAMING-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; CHECK-SME2p2-STREAMING-NEXT: cntp x8, p1, p1.b
+; CHECK-SME2p2-STREAMING-NEXT: compact z0.b, p1, z0.b
+; CHECK-SME2p2-STREAMING-NEXT: whilelo p0.b, xzr, x8
+; CHECK-SME2p2-STREAMING-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-SME2p2-STREAMING-NEXT: ret
+;
+; CHECK-STREAMING-COMPAT-LABEL: test_compressstore_v16i8:
+; CHECK-STREAMING-COMPAT: // %bb.0:
+; CHECK-STREAMING-COMPAT-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-STREAMING-COMPAT-NEXT: ptrue p0.b, vl16
+; CHECK-STREAMING-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-STREAMING-COMPAT-NEXT: lsl z1.b, z1.b, #7
+; CHECK-STREAMING-COMPAT-NEXT: asr z1.b, z1.b, #7
+; CHECK-STREAMING-COMPAT-NEXT: cmpne p1.b, p0/z, z1.b, #0
+; CHECK-STREAMING-COMPAT-NEXT: cntp x8, p1, p1.b
+; CHECK-STREAMING-COMPAT-NEXT: compact z0.b, p1, z0.b
+; CHECK-STREAMING-COMPAT-NEXT: whilelo p0.b, xzr, x8
+; CHECK-STREAMING-COMPAT-NEXT: st1b { z0.b }, p0, [x0]
+; CHECK-STREAMING-COMPAT-NEXT: ret
+ tail call void @llvm.masked.compressstore.v16i8(<16 x i8> %vec, ptr align 1 %p, <16 x i1> %mask)
+ ret void
+}
+
+define void @test_compressstore_v8f16(ptr %p, <8 x half> %vec, <8 x i1> %mask) {
+; CHECK-BASE-LABEL: test_compressstore_v8f16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-BASE-NEXT: ptrue p0.h, vl8
+; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-BASE-NEXT: shl v1.8h, v1.8h, #15
+; CHECK-BASE-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-BASE-NEXT: cntp x8, p1, p1.h
+; CHECK-BASE-NEXT: compact z0.h, p1, z0.h
+; CHECK-BASE-NEXT: whilelo p0.h, xzr, x8
+; CHECK-BASE-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-VL256-LABEL: test_compressstore_v8f16:
+; CHECK-VL256: // %bb.0:
+; CHECK-VL256-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-VL256-NEXT: ptrue p0.h, vl8
+; CHECK-VL256-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-VL256-NEXT: shl v1.8h, v1.8h, #15
+; CHECK-VL256-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-VL256-NEXT: cntp x8, p1, p1.h
+; CHECK-VL256-NEXT: compact z0.h, p1, z0.h
+; CHECK-VL256-NEXT: whilelo p0.h, xzr, x8
+; CHECK-VL256-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-VL256-NEXT: ret
+;
+; CHECK-SVE2p2-LABEL: test_compressstore_v8f16:
+; CHECK-SVE2p2: // %bb.0:
+; CHECK-SVE2p2-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-SVE2p2-NEXT: ptrue p0.h, vl8
+; CHECK-SVE2p2-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-SVE2p2-NEXT: shl v1.8h, v1.8h, #15
+; CHECK-SVE2p2-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SVE2p2-NEXT: cntp x8, p1, p1.h
+; CHECK-SVE2p2-NEXT: compact z0.h, p1, z0.h
+; CHECK-SVE2p2-NEXT: whilelo p0.h, xzr, x8
+; CHECK-SVE2p2-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-SVE2p2-NEXT: ret
+;
+; CHECK-SME2p2-STREAMING-LABEL: test_compressstore_v8f16:
+; CHECK-SME2p2-STREAMING: // %bb.0:
+; CHECK-SME2p2-STREAMING-NEXT: uunpklo z1.h, z1.b
+; CHECK-SME2p2-STREAMING-NEXT: ptrue p0.h, vl8
+; CHECK-SME2p2-STREAMING-NEXT: lsl z1.h, z1.h, #15
+; CHECK-SME2p2-STREAMING-NEXT: asr z1.h, z1.h, #15
+; CHECK-SME2p2-STREAMING-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SME2p2-STREAMING-NEXT: cntp x8, p1, p1.h
+; CHECK-SME2p2-STREAMING-NEXT: compact z0.h, p1, z0.h
+; CHECK-SME2p2-STREAMING-NEXT: whilelo p0.h, xzr, x8
+; CHECK-SME2p2-STREAMING-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-SME2p2-STREAMING-NEXT: ret
+;
+; CHECK-STREAMING-COMPAT-LABEL: test_compressstore_v8f16:
+; CHECK-STREAMING-COMPAT: // %bb.0:
+; CHECK-STREAMING-COMPAT-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-STREAMING-COMPAT-NEXT: ptrue p0.h, vl8
+; CHECK-STREAMING-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-STREAMING-COMPAT-NEXT: uunpklo z1.h, z1.b
+; CHECK-STREAMING-COMPAT-NEXT: lsl z1.h, z1.h, #15
+; CHECK-STREAMING-COMPAT-NEXT: asr z1.h, z1.h, #15
+; CHECK-STREAMING-COMPAT-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-STREAMING-COMPAT-NEXT: cntp x8, p1, p1.h
+; CHECK-STREAMING-COMPAT-NEXT: compact z0.h, p1, z0.h
+; CHECK-STREAMING-COMPAT-NEXT: whilelo p0.h, xzr, x8
+; CHECK-STREAMING-COMPAT-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-STREAMING-COMPAT-NEXT: ret
+ tail call void @llvm.masked.compressstore.v8f16(<8 x half> %vec, ptr align 1 %p, <8 x i1> %mask)
+ ret void
+}
+
+define void @test_compressstore_v8bf16(ptr %p, <8 x bfloat> %vec, <8 x i1> %mask) {
+; CHECK-BASE-LABEL: test_compressstore_v8bf16:
+; CHECK-BASE: // %bb.0:
+; CHECK-BASE-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-BASE-NEXT: ptrue p0.h, vl8
+; CHECK-BASE-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-BASE-NEXT: shl v1.8h, v1.8h, #15
+; CHECK-BASE-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-BASE-NEXT: cntp x8, p1, p1.h
+; CHECK-BASE-NEXT: compact z0.h, p1, z0.h
+; CHECK-BASE-NEXT: whilelo p0.h, xzr, x8
+; CHECK-BASE-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-VL256-LABEL: test_compressstore_v8bf16:
+; CHECK-VL256: // %bb.0:
+; CHECK-VL256-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-VL256-NEXT: ptrue p0.h, vl8
+; CHECK-VL256-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-VL256-NEXT: shl v1.8h, v1.8h, #15
+; CHECK-VL256-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-VL256-NEXT: cntp x8, p1, p1.h
+; CHECK-VL256-NEXT: compact z0.h, p1, z0.h
+; CHECK-VL256-NEXT: whilelo p0.h, xzr, x8
+; CHECK-VL256-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-VL256-NEXT: ret
+;
+; CHECK-SVE2p2-LABEL: test_compressstore_v8bf16:
+; CHECK-SVE2p2: // %bb.0:
+; CHECK-SVE2p2-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-SVE2p2-NEXT: ptrue p0.h, vl8
+; CHECK-SVE2p2-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-SVE2p2-NEXT: shl v1.8h, v1.8h, #15
+; CHECK-SVE2p2-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SVE2p2-NEXT: cntp x8, p1, p1.h
+; CHECK-SVE2p2-NEXT: compact z0.h, p1, z0.h
+; CHECK-SVE2p2-NEXT: whilelo p0.h, xzr, x8
+; CHECK-SVE2p2-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-SVE2p2-NEXT: ret
+;
+; CHECK-SME2p2-STREAMING-LABEL: test_compressstore_v8bf16:
+; CHECK-SME2p2-STREAMING: // %bb.0:
+; CHECK-SME2p2-STREAMING-NEXT: uunpklo z1.h, z1.b
+; CHECK-SME2p2-STREAMING-NEXT: ptrue p0.h, vl8
+; CHECK-SME2p2-STREAMING-NEXT: lsl z1.h, z1.h, #15
+; CHECK-SME2p2-STREAMING-NEXT: asr z1.h, z1.h, #15
+; CHECK-SME2p2-STREAMING-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-SME2p2-STREAMING-NEXT: cntp x8, p1, p1.h
+; CHECK-SME2p2-STREAMING-NEXT: compact z0.h, p1, z0.h
+; CHECK-SME2p2-STREAMING-NEXT: whilelo p0.h, xzr, x8
+; CHECK-SME2p2-STREAMING-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-SME2p2-STREAMING-NEXT: ret
+;
+; CHECK-STREAMING-COMPAT-LABEL: test_compressstore_v8bf16:
+; CHECK-STREAMING-COMPAT: // %bb.0:
+; CHECK-STREAMING-COMPAT-NEXT: // kill: def $d1 killed $d1 def $z1
+; CHECK-STREAMING-COMPAT-NEXT: ptrue p0.h, vl8
+; CHECK-STREAMING-COMPAT-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-STREAMING-COMPAT-NEXT: uunpklo z1.h, z1.b
+; CHECK-STREAMING-COMPAT-NEXT: lsl z1.h, z1.h, #15
+; CHECK-STREAMING-COMPAT-NEXT: asr z1.h, z1.h, #15
+; CHECK-STREAMING-COMPAT-NEXT: cmpne p1.h, p0/z, z1.h, #0
+; CHECK-STREAMING-COMPAT-NEXT: cntp x8, p1, p1.h
+; CHECK-STREAMING-COMPAT-NEXT: compact z0.h, p1, z0.h
+; CHECK-STREAMING-COMPAT-NEXT: whilelo p0.h, xzr, x8
+; CHECK-STREAMING-COMPAT-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-STREAMING-COMPAT-NEXT: ret
+ tail call void @llvm.masked.compressstore.v8bf16(<8 x bfloat> %vec, ptr align 1 %p, <8 x i1> %mask)
+ ret void
+}
>From e692e9928804b12a995bc31772aabd201a0dbb95 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Wed, 26 Aug 2026 14:45:32 +0100
Subject: [PATCH 7/9] Simplify Target Hook Check & Format
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 3 ++-
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h | 5 +++--
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index cdbf7fed84e10..4439d6b826ebb 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2227,7 +2227,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
if (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2()) {
// With +sve2p2/+sme2p2 the full range of vector types are supported.
- for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16}) {
+ for (auto VT :
+ {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv8f16, MVT::nxv8bf16}) {
setOperationAction(ISD::MSTORE, VT, Custom);
setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 5c7b2703b4257..71878619b6570 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -334,8 +334,9 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
}
bool isElementTypeLegalForCompressStore(Type *Ty) const {
- if ((ST->hasSVE2p2() || (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2())) &&
- (Ty->isIntegerTy(8) || Ty->isIntegerTy(16) || Ty->getScalarSizeInBits() == 16))
+ if ((ST->hasSVE2p2() ||
+ (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2())) &&
+ (Ty->isIntegerTy(8) || Ty->getScalarSizeInBits() == 16))
return true;
return Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isIntegerTy(32) ||
Ty->isIntegerTy(64);
>From 8c824d4675bfc1aa37ecaf127e1978d405b66b71 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 27 Aug 2026 10:05:30 +0100
Subject: [PATCH 8/9] Allow i8/i16 types in SME2p2 Streaming Mode
---
.../AArch64/AArch64TargetTransformInfo.h | 12 ++++----
.../CostModel/AArch64/masked_compress_load.ll | 30 +++++++++----------
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 71878619b6570..e11b4488d3e60 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -334,17 +334,17 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
}
bool isElementTypeLegalForCompressStore(Type *Ty) const {
- if ((ST->hasSVE2p2() ||
- (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2())) &&
- (Ty->isIntegerTy(8) || Ty->getScalarSizeInBits() == 16))
- return true;
- return Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isIntegerTy(32) ||
+ // For streaming SME2p2, only consider 8bit or 16bit Scalar types.
+ if ((ST->isStreamingSVEAvailable() && ST->hasSME2p2()))
+ return Ty->isIntegerTy(8) || Ty->getScalarSizeInBits() == 16;
+
+ return ((ST->hasSVE2p2() || ST->hasSME2p2()) && (Ty->isIntegerTy(8) || Ty->getScalarSizeInBits() == 16)) || Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isIntegerTy(32) ||
Ty->isIntegerTy(64);
}
bool isLegalMaskedCompressStore(Type *DataType,
Align Alignment) const override {
- if (!ST->isSVEorStreamingSVEAvailable())
+ if (!(ST->isSVEAvailable() || (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2())))
return false;
if (isa<FixedVectorType>(DataType) &&
diff --git a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
index 7cc880ce5b66d..69a3edb499b31 100644
--- a/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/masked_compress_load.ll
@@ -32,21 +32,21 @@ define void @fixed() {
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i8.p0(<2 x i8> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i8.p0(<4 x i8> poison, ptr poison, <4 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i8.p0(<8 x i8> poison, ptr poison, <8 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v16i8.p0(<16 x i8> poison, ptr poison, <16 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i16.p0(<2 x i16> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i16.p0(<4 x i16> poison, ptr poison, <4 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v8i16.p0(<8 x i16> poison, ptr poison, <8 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i32.p0(<2 x i32> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i32.p0(<4 x i32> poison, ptr poison, <4 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2i64.p0(<2 x i64> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f16.p0(<2 x half> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4f16.p0(<4 x half> poison, ptr poison, <4 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.v8f16.p0(<8 x half> poison, ptr poison, <8 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f32.p0(<2 x float> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4f32.p0(<4 x float> poison, ptr poison, <4 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v2f64.p0(<2 x double> poison, ptr poison, <2 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v4i64.p0(<4 x i64> poison, ptr poison, <4 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.v32f16.p0(<32 x half> poison, ptr poison, <32 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE2p2-SME2p2-LABEL: 'fixed'
@@ -164,25 +164,25 @@ define void @scalable() {
; SVE2p2-SME2p2-NON-STREAMING-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE2p2-SME2p2-STREAMING-LABEL: 'scalable'
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i8.p0(<vscale x 2 x i8> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i8.p0(<vscale x 4 x i8> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i8.p0(<vscale x 8 x i8> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv16i8.p0(<vscale x 16 x i8> poison, ptr poison, <vscale x 16 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2i16.p0(<vscale x 2 x i16> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4i16.p0(<vscale x 4 x i16> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8i16.p0(<vscale x 8 x i16> poison, ptr poison, <vscale x 8 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i32.p0(<vscale x 2 x i32> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i32.p0(<vscale x 4 x i32> poison, ptr poison, <vscale x 4 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2i64.p0(<vscale x 2 x i64> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv2f16.p0(<vscale x 2 x half> poison, ptr poison, <vscale x 2 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv4f16.p0(<vscale x 4 x half> poison, ptr poison, <vscale x 4 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.compressstore.nxv8f16.p0(<vscale x 8 x half> poison, ptr poison, <vscale x 8 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2f32.p0(<vscale x 2 x float> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4f32.p0(<vscale x 4 x float> poison, ptr poison, <vscale x 4 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv2f64.p0(<vscale x 2 x double> poison, ptr poison, <vscale x 2 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv1i64.p0(<vscale x 1 x i64> poison, ptr poison, <vscale x 1 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv4i64.p0(<vscale x 4 x i64> poison, ptr poison, <vscale x 4 x i1> poison)
-; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of Invalid for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
+; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.compressstore.nxv32f16.p0(<vscale x 32 x half> poison, ptr poison, <vscale x 32 x i1> poison)
; SVE2p2-SME2p2-STREAMING-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; SVE2p2-SME2p2-LABEL: 'scalable'
>From 3248b1571b3c524a277e2c329220a7da8c7fde18 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles at arm.com>
Date: Thu, 27 Aug 2026 10:06:36 +0100
Subject: [PATCH 9/9] format
---
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index e11b4488d3e60..2420e01eb3b3d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -338,13 +338,16 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
if ((ST->isStreamingSVEAvailable() && ST->hasSME2p2()))
return Ty->isIntegerTy(8) || Ty->getScalarSizeInBits() == 16;
- return ((ST->hasSVE2p2() || ST->hasSME2p2()) && (Ty->isIntegerTy(8) || Ty->getScalarSizeInBits() == 16)) || Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isIntegerTy(32) ||
+ return ((ST->hasSVE2p2() || ST->hasSME2p2()) &&
+ (Ty->isIntegerTy(8) || Ty->getScalarSizeInBits() == 16)) ||
+ Ty->isFloatTy() || Ty->isDoubleTy() || Ty->isIntegerTy(32) ||
Ty->isIntegerTy(64);
}
bool isLegalMaskedCompressStore(Type *DataType,
Align Alignment) const override {
- if (!(ST->isSVEAvailable() || (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2())))
+ if (!(ST->isSVEAvailable() ||
+ (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2())))
return false;
if (isa<FixedVectorType>(DataType) &&
More information about the llvm-commits
mailing list