[llvm] [AArch64][ISel] Enable factor 3 interleave/deinterleave i1 support (PR #215524)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 16 15:50:57 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/215524
>From 98de36aca559603e2d9c17f6a1d64743598856f7 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 11 Aug 2026 11:02:54 +0000
Subject: [PATCH] [AArch64][ISel] Enable factor 3 interleave/deinterleave i1
support
Add support for factor 3 interleave/deinterleave SVE predicate vectors.
This works by converting predicate operands to packed integer vectors,
reusing the existing interleave/deinterleave lowering, then converting
the nonzero results back to predicates.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 30 +++++
.../AArch64/sve-vector-deinterleave.ll | 52 +++++++++
.../CodeGen/AArch64/sve-vector-interleave.ll | 108 ++++++++++++++++++
3 files changed, 190 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 6003db1c72449..acda7ce3ef3ff 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -34503,6 +34503,30 @@ AArch64TargetLowering::LowerFixedLengthIntToFPToSVE(SDValue Op,
}
}
+static SDValue lowerPredicateInterleaveWithPackedVectors(SDValue Op,
+ SelectionDAG &DAG) {
+ SDLoc DL(Op);
+ EVT OpVT = Op.getValueType();
+ assert(OpVT.isScalableVector() && OpVT.getVectorElementType() == MVT::i1 &&
+ Op->getNumOperands() == 3 && "Unexpected predicate interleave");
+
+ EVT PackedVT = getPackedSVEVectorVT(OpVT.getVectorElementCount());
+ SmallVector<SDValue, 3> PackedOps;
+ for (SDValue V : Op->ops())
+ PackedOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, PackedVT, V));
+
+ SmallVector<EVT, 3> PackedVTs(3, PackedVT);
+ SDValue PackedOp =
+ DAG.getNode(Op.getOpcode(), DL, DAG.getVTList(PackedVTs), PackedOps);
+
+ SDValue Zero = DAG.getConstant(0, DL, PackedVT);
+ SmallVector<SDValue, 3> Results;
+ for (unsigned I = 0; I != 3; ++I)
+ Results.push_back(
+ DAG.getSetCC(DL, OpVT, PackedOp.getValue(I), Zero, ISD::SETNE));
+ return DAG.getMergeValues(Results, DL);
+}
+
SDValue
AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op,
SelectionDAG &DAG) const {
@@ -34541,6 +34565,9 @@ AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op,
}
if (OpVT.isScalableVector() && Op->getNumOperands() == 3) {
+ if (OpVT.getVectorElementType() == MVT::i1)
+ return lowerPredicateInterleaveWithPackedVectors(Op, DAG);
+
// aarch64_sve_ld3 only supports packed datatypes.
EVT PackedVT = getPackedSVEVectorVT(OpVT.getVectorElementCount());
Align Alignment = DAG.getReducedAlign(PackedVT, /*UseABI=*/false);
@@ -34651,6 +34678,9 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
}
if (OpVT.isScalableVector() && Op->getNumOperands() == 3) {
+ if (OpVT.getVectorElementType() == MVT::i1)
+ return lowerPredicateInterleaveWithPackedVectors(Op, DAG);
+
// aarch64_sve_st3 only supports packed datatypes.
EVT PackedVT = getPackedSVEVectorVT(OpVT.getVectorElementCount());
SmallVector<SDValue, 3> InVecs;
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
index f0d4e86752dec..560f86cb052dc 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
@@ -712,6 +712,58 @@ define {<vscale x 2 x i1>, <vscale x 2 x i1>} @vector_deinterleave_nxv2i1_nxv4i1
ret {<vscale x 2 x i1>, <vscale x 2 x i1>} %retval
}
+define {<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>} @deinterleave3_nxv8i1_nxv24i1(<vscale x 24 x i1> %vec) {
+; CHECK-LABEL: deinterleave3_nxv16i1_nxv24i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: punpklo p1.h, p1.b
+; CHECK-NEXT: punpkhi p3.h, p0.b
+; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: mov z0.h, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z1.h, p3/z, #1 // =0x1
+; CHECK-NEXT: ptrue p2.h
+; CHECK-NEXT: mov z2.h, p0/z, #1 // =0x1
+; CHECK-NEXT: str z0, [sp, #2, mul vl]
+; CHECK-NEXT: str z1, [sp, #1, mul vl]
+; CHECK-NEXT: str z2, [sp]
+; CHECK-NEXT: ld3h { z0.h - z2.h }, p2/z, [sp]
+; CHECK-NEXT: cmpne p0.h, p2/z, z0.h, #0
+; CHECK-NEXT: cmpne p1.h, p2/z, z1.h, #0
+; CHECK-NEXT: cmpne p2.h, p2/z, z2.h, #0
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>} @llvm.vector.deinterleave3.nxv24i1(<vscale x 24 x i1> %vec)
+ ret {<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>} %retval
+}
+
+define {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @deinterleave3_nxv16i1_nxv48i1(<vscale x 48 x i1> %vec) {
+; CHECK-LABEL: deinterleave3_nxv16i1_nxv48i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z0.b, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z1.b, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z2.b, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p2.b
+; CHECK-NEXT: str z0, [sp, #2, mul vl]
+; CHECK-NEXT: str z1, [sp, #1, mul vl]
+; CHECK-NEXT: str z2, [sp]
+; CHECK-NEXT: ld3b { z0.b - z2.b }, p2/z, [sp]
+; CHECK-NEXT: cmpne p0.b, p2/z, z0.b, #0
+; CHECK-NEXT: cmpne p1.b, p2/z, z1.b, #0
+; CHECK-NEXT: cmpne p2.b, p2/z, z2.b, #0
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.vector.deinterleave3.nxv48i1(<vscale x 48 x i1> %vec)
+ ret {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} %retval
+}
; Split illegal types
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
index c497ba6f7008b..32689d88c09ea 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
@@ -659,6 +659,114 @@ define <vscale x 2 x i1> @interleave2_nxv2i1(<vscale x 1 x i1> %vec0, <vscale x
ret <vscale x 2 x i1> %retval
}
+define <vscale x 6 x i1> @interleave3_nxv6i1(<vscale x 2 x i1> %vec0, <vscale x 2 x i1> %vec1, <vscale x 2 x i1> %vec2) {
+; CHECK-LABEL: interleave3_nxv6i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z2.d, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z1.d, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [sp]
+; CHECK-NEXT: ldr z0, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z1, [sp]
+; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0
+; CHECK-NEXT: cmpne p2.d, p0/z, z1.d, #0
+; CHECK-NEXT: cmpne p3.d, p0/z, z2.d, #0
+; CHECK-NEXT: uzp1 p0.s, p2.s, p1.s
+; CHECK-NEXT: uzp1 p1.s, p3.s, p0.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call <vscale x 6 x i1> @llvm.vector.interleave3.nxv6i1(<vscale x 2 x i1> %vec0, <vscale x 2 x i1> %vec1, <vscale x 2 x i1> %vec2)
+ ret <vscale x 6 x i1> %retval
+}
+
+define <vscale x 12 x i1> @interleave3_nxv12i1(<vscale x 4 x i1> %vec0, <vscale x 4 x i1> %vec1, <vscale x 4 x i1> %vec2) {
+; CHECK-LABEL: interleave3_nxv12i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z2.s, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z1.s, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [sp]
+; CHECK-NEXT: ldr z0, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z1, [sp]
+; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0
+; CHECK-NEXT: cmpne p2.s, p0/z, z1.s, #0
+; CHECK-NEXT: cmpne p3.s, p0/z, z2.s, #0
+; CHECK-NEXT: uzp1 p0.h, p2.h, p1.h
+; CHECK-NEXT: uzp1 p1.h, p3.h, p0.h
+; CHECK-NEXT: uzp1 p0.b, p0.b, p1.b
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call <vscale x 12 x i1> @llvm.vector.interleave3.nxv12i1(<vscale x 4 x i1> %vec0, <vscale x 4 x i1> %vec1, <vscale x 4 x i1> %vec2)
+ ret <vscale x 12 x i1> %retval
+}
+
+define <vscale x 24 x i1> @interleave3_nxv24i1(<vscale x 8 x i1> %vec0, <vscale x 8 x i1> %vec1, <vscale x 8 x i1> %vec2) {
+; CHECK-LABEL: interleave3_nxv24i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z2.h, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z1.h, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: st3h { z0.h - z2.h }, p0, [sp]
+; CHECK-NEXT: ldr z0, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z1, [sp]
+; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: cmpne p2.h, p0/z, z1.h, #0
+; CHECK-NEXT: cmpne p3.h, p0/z, z2.h, #0
+; CHECK-NEXT: uzp1 p0.b, p2.b, p1.b
+; CHECK-NEXT: uzp1 p1.b, p3.b, p0.b
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call <vscale x 24 x i1> @llvm.vector.interleave3.nxv24i1(<vscale x 8 x i1> %vec0, <vscale x 8 x i1> %vec1, <vscale x 8 x i1> %vec2)
+ ret <vscale x 24 x i1> %retval
+}
+
+define <vscale x 48 x i1> @interleave3_nxv48i1(<vscale x 16 x i1> %vec0, <vscale x 16 x i1> %vec1, <vscale x 16 x i1> %vec2) {
+; CHECK-LABEL: interleave3_nxv48i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z2.b, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z1.b, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p2.b
+; CHECK-NEXT: st3b { z0.b - z2.b }, p2, [sp]
+; CHECK-NEXT: ldr z0, [sp]
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: cmpne p0.b, p2/z, z0.b, #0
+; CHECK-NEXT: cmpne p1.b, p2/z, z1.b, #0
+; CHECK-NEXT: cmpne p2.b, p2/z, z2.b, #0
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %vec0, <vscale x 16 x i1> %vec1, <vscale x 16 x i1> %vec2)
+ ret <vscale x 48 x i1> %retval
+}
+
; Split illegal type size
define <vscale x 16 x i32> @interleave2_nxv16i32(<vscale x 8 x i32> %vec0, <vscale x 8 x i32> %vec1) {
More information about the llvm-commits
mailing list