[llvm] [AArch64][ISel] Support i1 masked interleaved loads and stores (PR #209231)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 01:25:11 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/209231
>From 5146a92d7618608ec4386d40c8a0567a946a6bfc Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Mon, 13 Jul 2026 15:13:30 +0000
Subject: [PATCH] [AArch64][ISel] Support i1 masked interleaved loads and
stores
Lower predicate value interleaved memory operations through full-width
integer containers. Extend predicates before st2/st4 and reconstruct
them with comparisons after ld2/ld4.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 55 ++++++--
.../scalable_masked_deinterleaved_loads.ll | 120 ++++++++++++++++++
.../scalable_masked_interleaved_stores.ll | 112 ++++++++++++++++
3 files changed, 276 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 93b4b33043755..c51387a7067d9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27538,6 +27538,38 @@ static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL,
WideMask->getOperand(0));
}
+static bool isSupportedSVEInterleavedMemSubvectorType(EVT VT,
+ SelectionDAG &DAG) {
+ return VT.isScalableVector() &&
+ (VT.getSizeInBits().getKnownMinValue() == 128 ||
+ (VT.isVectorOf(MVT::i1) && VT != MVT::nxv1i1)) &&
+ DAG.getTargetLoweringInfo().isTypeLegal(VT);
+}
+
+static EVT getSVEInterleavedMemContainerType(EVT VT) {
+ return VT.isVectorOf(MVT::i1) ? getPromotedVTForPredicate(VT) : VT;
+}
+
+static SDValue convertToSVEInterleavedMemContainer(SDValue Value, SDLoc DL,
+ SelectionDAG &DAG) {
+ EVT ContainerVT = getSVEInterleavedMemContainerType(Value.getValueType());
+ if (Value.getValueType() == ContainerVT)
+ return Value;
+ return DAG.getNode(ISD::ZERO_EXTEND, DL, ContainerVT, Value);
+}
+
+static SDValue convertFromSVEInterleavedMemContainer(SDValue Value, EVT VT,
+ SDLoc DL,
+ SelectionDAG &DAG) {
+ if (Value.getValueType() == VT)
+ return Value;
+ assert(VT.isVectorOf(MVT::i1) &&
+ Value.getValueType() == getPromotedVTForPredicate(VT) &&
+ "Unexpected SVE interleaved memory container conversion");
+ return DAG.getSetCC(DL, VT, Value,
+ DAG.getConstant(0, DL, Value.getValueType()), ISD::SETNE);
+}
+
static SDValue performInterleavedMaskedStoreCombine(
SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) {
if (!DCI.isBeforeLegalize())
@@ -27565,9 +27597,7 @@ static SDValue performInterleavedMaskedStoreCombine(
// At the moment we're unlikely to see a fixed-width vector interleave as
// we usually generate shuffles instead.
EVT SubVecTy = ValueInterleaveOps[0].getValueType();
- if (!SubVecTy.isScalableVT() ||
- SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
- !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+ if (!isSupportedSVEInterleavedMemSubvectorType(SubVecTy, DAG))
return SDValue();
SDLoc DL(N);
@@ -27576,6 +27606,9 @@ static SDValue performInterleavedMaskedStoreCombine(
if (!NarrowMask)
return SDValue();
+ for (SDValue &Value : ValueInterleaveOps)
+ Value = convertToSVEInterleavedMemContainer(Value, DL, DAG);
+
const Intrinsic::ID IID =
NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4;
SmallVector<SDValue, 8> NewStOps;
@@ -30473,9 +30506,7 @@ static SDValue performVectorDeinterleaveCombine(
// At the moment we're unlikely to see a fixed-width vector deinterleave as
// we usually generate shuffles instead.
unsigned MinNumElements = SubVecTy.getVectorMinNumElements();
- if (!SubVecTy.isScalableVector() ||
- SubVecTy.getSizeInBits().getKnownMinValue() != 128 ||
- !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy))
+ if (!isSupportedSVEInterleavedMemSubvectorType(SubVecTy, DAG))
return SDValue();
// Make sure each input operand is the correct extract_subvector of the same
@@ -30511,6 +30542,8 @@ static SDValue performVectorDeinterleaveCombine(
if (!NarrowMask)
return SDValue();
+ EVT LoadVT = getSVEInterleavedMemContainerType(SubVecTy);
+
const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret
: Intrinsic::aarch64_sve_ld4_sret;
SDValue NewLdOps[] = {MaskedLoad->getChain(),
@@ -30518,17 +30551,17 @@ static SDValue performVectorDeinterleaveCombine(
MaskedLoad->getBasePtr()};
SDValue Res;
if (NumParts == 2)
- Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
- {SubVecTy, SubVecTy, MVT::Other}, NewLdOps);
+ Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {LoadVT, LoadVT, MVT::Other},
+ NewLdOps);
else
Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
- {SubVecTy, SubVecTy, SubVecTy, SubVecTy, MVT::Other},
- NewLdOps);
+ {LoadVT, LoadVT, LoadVT, LoadVT, MVT::Other}, NewLdOps);
// We can now generate a structured load!
SmallVector<SDValue, 4> ResOps(NumParts);
for (unsigned Idx = 0; Idx < NumParts; Idx++)
- ResOps[Idx] = SDValue(Res.getNode(), Idx);
+ ResOps[Idx] = convertFromSVEInterleavedMemContainer(
+ SDValue(Res.getNode(), Idx), SubVecTy, DL, DAG);
// Replace uses of the original chain result with the new chain result.
DAG.ReplaceAllUsesOfValueWith(SDValue(MaskedLoad, 1),
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index ee2d482d9ffb5..ca5bae6791cf6 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -1,6 +1,62 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+define { <vscale x 2 x i1>, <vscale x 2 x i1> } @foo_ld2_nxv2i1(<vscale x 2 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld2_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0]
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: cmpne p0.d, p1/z, z0.d, #0
+; CHECK-NEXT: cmpne p1.d, p1/z, z1.d, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %wide.masked.vec = call <vscale x 4 x i1> @llvm.masked.load.nxv4i1.p0(ptr %p, i32 1, <vscale x 4 x i1> %interleaved.mask, <vscale x 4 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 2 x i1>, <vscale x 2 x i1> } @llvm.vector.deinterleave2.nxv4i1(<vscale x 4 x i1> %wide.masked.vec)
+ ret { <vscale x 2 x i1>, <vscale x 2 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 4 x i1>, <vscale x 4 x i1> } @foo_ld2_nxv4i1(<vscale x 4 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld2_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld2w { z0.s, z1.s }, p0/z, [x0]
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: cmpne p0.s, p1/z, z0.s, #0
+; CHECK-NEXT: cmpne p1.s, p1/z, z1.s, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %wide.masked.vec = call <vscale x 8 x i1> @llvm.masked.load.nxv8i1.p0(ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask, <vscale x 8 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 4 x i1>, <vscale x 4 x i1> } @llvm.vector.deinterleave2.nxv8i1(<vscale x 8 x i1> %wide.masked.vec)
+ ret { <vscale x 4 x i1>, <vscale x 4 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 8 x i1>, <vscale x 8 x i1> } @foo_ld2_nxv8i1(<vscale x 8 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld2_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0]
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: cmpne p0.h, p1/z, z0.h, #0
+; CHECK-NEXT: cmpne p1.h, p1/z, z1.h, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %wide.masked.vec = call <vscale x 16 x i1> @llvm.masked.load.nxv16i1.p0(ptr %p, i32 1, <vscale x 16 x i1> %interleaved.mask, <vscale x 16 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 8 x i1>, <vscale x 8 x i1> } @llvm.vector.deinterleave2.nxv16i1(<vscale x 16 x i1> %wide.masked.vec)
+ ret { <vscale x 8 x i1>, <vscale x 8 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 16 x i1>, <vscale x 16 x i1> } @foo_ld2_nxv16i1(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld2_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0]
+; CHECK-NEXT: ptrue p1.b
+; CHECK-NEXT: cmpne p0.b, p1/z, z0.b, #0
+; CHECK-NEXT: cmpne p1.b, p1/z, z1.b, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %wide.masked.vec = call <vscale x 32 x i1> @llvm.masked.load.nxv32i1.p0(ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask, <vscale x 32 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i1>, <vscale x 16 x i1> } @llvm.vector.deinterleave2.nxv32i1(<vscale x 32 x i1> %wide.masked.vec)
+ ret { <vscale x 16 x i1>, <vscale x 16 x i1> } %deinterleaved.vec
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld2_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld2_nxv16i8:
; CHECK: // %bb.0:
@@ -45,6 +101,70 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @foo_ld2_nxv2f64(<vscale
ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleaved.vec
}
+define { <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1> } @foo_ld4_nxv2i1(<vscale x 2 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld4d { z0.d - z3.d }, p0/z, [x0]
+; CHECK-NEXT: ptrue p3.d
+; CHECK-NEXT: cmpne p0.d, p3/z, z0.d, #0
+; CHECK-NEXT: cmpne p1.d, p3/z, z1.d, #0
+; CHECK-NEXT: cmpne p2.d, p3/z, z2.d, #0
+; CHECK-NEXT: cmpne p3.d, p3/z, z3.d, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %wide.masked.vec = call <vscale x 8 x i1> @llvm.masked.load.nxv8i1.p0(ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask, <vscale x 8 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1> } @llvm.vector.deinterleave4.nxv8i1(<vscale x 8 x i1> %wide.masked.vec)
+ ret { <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1>, <vscale x 2 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1> } @foo_ld4_nxv4i1(<vscale x 4 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld4w { z0.s - z3.s }, p0/z, [x0]
+; CHECK-NEXT: ptrue p3.s
+; CHECK-NEXT: cmpne p0.s, p3/z, z0.s, #0
+; CHECK-NEXT: cmpne p1.s, p3/z, z1.s, #0
+; CHECK-NEXT: cmpne p2.s, p3/z, z2.s, #0
+; CHECK-NEXT: cmpne p3.s, p3/z, z3.s, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %wide.masked.vec = call <vscale x 16 x i1> @llvm.masked.load.nxv16i1.p0(ptr %p, i32 1, <vscale x 16 x i1> %interleaved.mask, <vscale x 16 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1> } @llvm.vector.deinterleave4.nxv16i1(<vscale x 16 x i1> %wide.masked.vec)
+ ret { <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1>, <vscale x 4 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1> } @foo_ld4_nxv8i1(<vscale x 8 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x0]
+; CHECK-NEXT: ptrue p3.h
+; CHECK-NEXT: cmpne p0.h, p3/z, z0.h, #0
+; CHECK-NEXT: cmpne p1.h, p3/z, z1.h, #0
+; CHECK-NEXT: cmpne p2.h, p3/z, z2.h, #0
+; CHECK-NEXT: cmpne p3.h, p3/z, z3.h, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave4.nxv32i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %wide.masked.vec = call <vscale x 32 x i1> @llvm.masked.load.nxv32i1.p0(ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask, <vscale x 32 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1> } @llvm.vector.deinterleave4.nxv32i1(<vscale x 32 x i1> %wide.masked.vec)
+ ret { <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1> } %deinterleaved.vec
+}
+
+define { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } @foo_ld4_nxv16i1(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld4_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0]
+; CHECK-NEXT: ptrue p3.b
+; CHECK-NEXT: cmpne p0.b, p3/z, z0.b, #0
+; CHECK-NEXT: cmpne p1.b, p3/z, z1.b, #0
+; CHECK-NEXT: cmpne p2.b, p3/z, z2.b, #0
+; CHECK-NEXT: cmpne p3.b, p3/z, z3.b, #0
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %wide.masked.vec = call <vscale x 64 x i1> @llvm.masked.load.nxv64i1.p0(ptr %p, i32 1, <vscale x 64 x i1> %interleaved.mask, <vscale x 64 x i1> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } @llvm.vector.deinterleave4.nxv64i1(<vscale x 64 x i1> %wide.masked.vec)
+ ret { <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1> } %deinterleaved.vec
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld4_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld4_nxv16i8:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index d3cd9bf08cc0a..17fb609144702 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -1,6 +1,58 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+define void @foo_st2_nxv2i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %val1, <vscale x 2 x i1> %val2, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z0.d, p1/z, #1 // =0x1
+; CHECK-NEXT: st2d { z0.d, z1.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %interleaved.value = call <vscale x 4 x i1> @llvm.vector.interleave2.nxv4i1(<vscale x 2 x i1> %val1, <vscale x 2 x i1> %val2)
+ call void @llvm.masked.store.nxv4i1.p0(<vscale x 4 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 4 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st2_nxv4i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %val1, <vscale x 4 x i1> %val2, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.s, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z0.s, p1/z, #1 // =0x1
+; CHECK-NEXT: st2w { z0.s, z1.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %interleaved.value = call <vscale x 8 x i1> @llvm.vector.interleave2.nxv8i1(<vscale x 4 x i1> %val1, <vscale x 4 x i1> %val2)
+ call void @llvm.masked.store.nxv8i1.p0(<vscale x 8 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st2_nxv8i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %val1, <vscale x 8 x i1> %val2, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.h, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z0.h, p1/z, #1 // =0x1
+; CHECK-NEXT: st2h { z0.h, z1.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %interleaved.value = call <vscale x 16 x i1> @llvm.vector.interleave2.nxv16i1(<vscale x 8 x i1> %val1, <vscale x 8 x i1> %val2)
+ call void @llvm.masked.store.nxv16i1.p0(<vscale x 16 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 16 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st2_nxv16i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %val1, <vscale x 16 x i1> %val2, ptr %p) {
+; CHECK-LABEL: foo_st2_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.b, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
+; CHECK-NEXT: st2b { z0.b, z1.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %interleaved.value = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %val1, <vscale x 16 x i1> %val2)
+ call void @llvm.masked.store.nxv32i1.p0(<vscale x 32 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask)
+ ret void
+}
+
define void @foo_st2_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, ptr %p) {
; CHECK-LABEL: foo_st2_nxv16i8:
; CHECK: // %bb.0:
@@ -68,6 +120,66 @@ define void @foo_st4_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1,
ret void
}
+define void @foo_st4_nxv2i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %v, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv2i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.d, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4d { z0.d - z3.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %interleaved.value = call <vscale x 8 x i1> @llvm.vector.interleave4.nxv8i1(<vscale x 2 x i1> %v, <vscale x 2 x i1> %v, <vscale x 2 x i1> %v, <vscale x 2 x i1> %v)
+ call void @llvm.masked.store.nxv8i1.p0(<vscale x 8 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 8 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st4_nxv4i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %v, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv4i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.s, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4w { z0.s - z3.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %interleaved.value = call <vscale x 16 x i1> @llvm.vector.interleave4.nxv16i1(<vscale x 4 x i1> %v, <vscale x 4 x i1> %v, <vscale x 4 x i1> %v, <vscale x 4 x i1> %v)
+ call void @llvm.masked.store.nxv16i1.p0(<vscale x 16 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 16 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st4_nxv8i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %v, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv8i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.h, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4h { z0.h - z3.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 32 x i1> @llvm.vector.interleave4.nxv32i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %interleaved.value = call <vscale x 32 x i1> @llvm.vector.interleave4.nxv32i1(<vscale x 8 x i1> %v, <vscale x 8 x i1> %v, <vscale x 8 x i1> %v, <vscale x 8 x i1> %v)
+ call void @llvm.masked.store.nxv32i1.p0(<vscale x 32 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 32 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st4_nxv16i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %v, ptr %p) {
+; CHECK-LABEL: foo_st4_nxv16i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z0.d
+; CHECK-NEXT: st4b { z0.b - z3.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %interleaved.value = call <vscale x 64 x i1> @llvm.vector.interleave4.nxv64i1(<vscale x 16 x i1> %v, <vscale x 16 x i1> %v, <vscale x 16 x i1> %v, <vscale x 16 x i1> %v)
+ call void @llvm.masked.store.nxv64i1.p0(<vscale x 64 x i1> %interleaved.value, ptr %p, i32 1, <vscale x 64 x i1> %interleaved.mask)
+ ret void
+}
+
define void @foo_st4_nxv8i16(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %val1, <vscale x 8 x i16> %val2, <vscale x 8 x i16> %val3, <vscale x 8 x i16> %val4, ptr %p) {
; CHECK-LABEL: foo_st4_nxv8i16:
; CHECK: // %bb.0:
More information about the llvm-commits
mailing list