[llvm] [AArch64][SVE] Enable factor-3 scalable interleave operations (PR #200424)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 03:03:49 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/200424
>From 64db40e78de0210e8d018222a68f719457e59c61 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Sun, 2 Aug 2026 22:48:33 +0000
Subject: [PATCH] [AArch64][ISel] Enable factor-3 scalable interleave
operations
Enable factor-3 scalable vector interleave and deinterleave lowering for SVE,
including the corresponding interleaved load/store combines.
Factor-3 operations are lowered through the existing LD3/ST3-based path, which
operates on packed data vectors.
Unlike factors 2 and 4, SVE has no ZIP/UZP instruction sequence specifically
ror factor-3 operations. For LD3/ST3 predicate operands must be promoted
into Z-register data vectors before lowering, then convert the results
back to predicates by comparing against zero. This avoids invalid
predicate uses of getSVESafeBitCast.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 64 ++++++++++++++++---
.../scalable_masked_deinterleaved_loads.ll | 11 ++++
.../scalable_masked_interleaved_stores.ll | 14 ++++
.../AArch64/sve-vector-deinterleave.ll | 52 +++++++++++++++
.../CodeGen/AArch64/sve-vector-interleave.ll | 52 +++++++++++++++
5 files changed, 185 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 51be0e66b19b0..802a16778d022 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27784,8 +27784,6 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
auto *MemN = cast<MemSDNode>(N);
if (IsScalable) {
- if (NumParts == 3)
- return SDValue();
SDValue Pred;
if (IsMasked) {
Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts);
@@ -27796,8 +27794,18 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
Pred = DAG.getConstant(1, DL, PredVT);
}
- const Intrinsic::ID IID =
- NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4;
+ Intrinsic::ID IID;
+ switch (NumParts) {
+ case 2:
+ IID = Intrinsic::aarch64_sve_st2;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_sve_st3;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_sve_st4;
+ break;
+ }
SmallVector<SDValue, 8> Ops;
Ops.append({Chain, DAG.getConstant(IID, DL, MVT::i32)});
Ops.append(ValueInterleaveOps);
@@ -30848,8 +30856,6 @@ static SDValue performVectorDeinterleaveCombine(
SDValue Res;
MemSDNode *MemNode = dyn_cast<MemSDNode>(WideVec);
if (IsScalable) {
- if (NumParts == 3)
- return SDValue();
SDValue Chain, BasePtr, Pred;
if (auto *MaskedLoad = dyn_cast<MaskedLoadSDNode>(WideVec)) {
// Bail out if the masked load has an unexpected number of uses, since we
@@ -30883,8 +30889,18 @@ static SDValue performVectorDeinterleaveCombine(
BasePtr = Load->getBasePtr();
}
- const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret
- : Intrinsic::aarch64_sve_ld4_sret;
+ Intrinsic::ID IID;
+ switch (NumParts) {
+ case 2:
+ IID = Intrinsic::aarch64_sve_ld2_sret;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_sve_ld3_sret;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_sve_ld4_sret;
+ break;
+ }
SDValue NewLdOps[] = {Chain, DAG.getConstant(IID, DL, MVT::i32), Pred,
BasePtr};
Res = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, ResVTList,
@@ -34352,12 +34368,40 @@ AArch64TargetLowering::LowerFixedLengthIntToFPToSVE(SDValue Op,
}
}
+static SDValue lowerPredicateInterleaveWithPackedVectors(SDValue Op,
+ SelectionDAG &DAG) {
+ SDLoc DL(Op);
+ EVT OpVT = Op.getValueType();
+ assert(OpVT.isScalableVector() && OpVT.getVectorElementType() == MVT::i1 &&
+ Op->getNumOperands() == 3 && "Unexpected predicate interleave");
+
+ EVT PackedVT = getPackedSVEVectorVT(OpVT.getVectorElementCount());
+ SmallVector<SDValue, 3> PackedOps;
+ for (SDValue V : Op->ops())
+ PackedOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, PackedVT, V));
+
+ SmallVector<EVT, 3> PackedVTs(Op->getNumValues(), PackedVT);
+ SDValue PackedOp =
+ DAG.getNode(Op.getOpcode(), DL, DAG.getVTList(PackedVTs), PackedOps);
+
+ SDValue Zero = DAG.getConstant(0, DL, PackedVT);
+ SmallVector<SDValue, 3> Results;
+ for (unsigned I = 0; I != Op->getNumValues(); ++I)
+ Results.push_back(
+ DAG.getSetCC(DL, OpVT, PackedOp.getValue(I), Zero, ISD::SETNE));
+ return DAG.getMergeValues(Results, DL);
+}
+
SDValue
AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op,
SelectionDAG &DAG) const {
SDLoc DL(Op);
EVT OpVT = Op.getValueType();
+ if (OpVT.isScalableVector() && OpVT.getVectorElementType() == MVT::i1 &&
+ Op->getNumOperands() == 3)
+ return lowerPredicateInterleaveWithPackedVectors(Op, DAG);
+
if (OpVT.isFixedLengthVector() && Op->getNumOperands() == 3) {
Align Alignment = DAG.getReducedAlign(OpVT, /*UseABI=*/false);
SDValue StackPtr =
@@ -34468,6 +34512,10 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op,
SDLoc DL(Op);
EVT OpVT = Op.getValueType();
+ if (OpVT.isScalableVector() && OpVT.getVectorElementType() == MVT::i1 &&
+ Op->getNumOperands() == 3)
+ return lowerPredicateInterleaveWithPackedVectors(Op, DAG);
+
if (OpVT.isFixedLengthVector() && Op->getNumOperands() == 3) {
Align Alignment = DAG.getReducedAlign(OpVT, /*UseABI=*/false);
SDValue StackPtr =
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index ee2d482d9ffb5..031509d5ff6c3 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -45,6 +45,17 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @foo_ld2_nxv2f64(<vscale
ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleaved.vec
}
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld3_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld3_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %wide.masked.vec = call <vscale x 48 x i8> @llvm.masked.load.nxv48i8(ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask, <vscale x 48 x i8> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave3.nxv48i8(<vscale x 48 x i8> %wide.masked.vec)
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.vec
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld4_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld4_nxv16i8:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index fc2f64b756959..88c82b222bc09 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -54,6 +54,20 @@ define void @foo_st2_nxv2i64(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %val1,
ret void
}
+define void @foo_st3_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %interleaved.value = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3)
+ call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask)
+ ret void
+}
+
define void @foo_st4_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, <vscale x 16 x i8> %val4, ptr %p) {
; CHECK-LABEL: foo_st4_nxv16i8:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
index f0d4e86752dec..62d62de9e3971 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll
@@ -712,6 +712,58 @@ define {<vscale x 2 x i1>, <vscale x 2 x i1>} @vector_deinterleave_nxv2i1_nxv4i1
ret {<vscale x 2 x i1>, <vscale x 2 x i1>} %retval
}
+define {<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>} @deinterleave3_nxv16i1_nxv24i1(<vscale x 24 x i1> %vec) {
+; CHECK-LABEL: deinterleave3_nxv16i1_nxv24i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: punpklo p1.h, p1.b
+; CHECK-NEXT: punpkhi p3.h, p0.b
+; CHECK-NEXT: punpklo p0.h, p0.b
+; CHECK-NEXT: mov z0.h, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z1.h, p3/z, #1 // =0x1
+; CHECK-NEXT: ptrue p2.h
+; CHECK-NEXT: mov z2.h, p0/z, #1 // =0x1
+; CHECK-NEXT: str z0, [sp, #2, mul vl]
+; CHECK-NEXT: str z1, [sp, #1, mul vl]
+; CHECK-NEXT: str z2, [sp]
+; CHECK-NEXT: ld3h { z0.h - z2.h }, p2/z, [sp]
+; CHECK-NEXT: cmpne p0.h, p2/z, z0.h, #0
+; CHECK-NEXT: cmpne p1.h, p2/z, z1.h, #0
+; CHECK-NEXT: cmpne p2.h, p2/z, z2.h, #0
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>} @llvm.vector.deinterleave3.nxv24i1(<vscale x 24 x i1> %vec)
+ ret {<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>} %retval
+}
+
+define {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @deinterleave3_nxv16i1_nxv48i1(<vscale x 48 x i1> %vec) {
+; CHECK-LABEL: deinterleave3_nxv16i1_nxv48i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z0.b, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z1.b, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z2.b, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p2.b
+; CHECK-NEXT: str z0, [sp, #2, mul vl]
+; CHECK-NEXT: str z1, [sp, #1, mul vl]
+; CHECK-NEXT: str z2, [sp]
+; CHECK-NEXT: ld3b { z0.b - z2.b }, p2/z, [sp]
+; CHECK-NEXT: cmpne p0.b, p2/z, z0.b, #0
+; CHECK-NEXT: cmpne p1.b, p2/z, z1.b, #0
+; CHECK-NEXT: cmpne p2.b, p2/z, z2.b, #0
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.vector.deinterleave3.nxv48i1(<vscale x 48 x i1> %vec)
+ ret {<vscale x 16 x i1>, <vscale x 16 x i1>, <vscale x 16 x i1>} %retval
+}
; Split illegal types
diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
index c497ba6f7008b..dcb874852fd3e 100644
--- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll
@@ -659,6 +659,58 @@ define <vscale x 2 x i1> @interleave2_nxv2i1(<vscale x 1 x i1> %vec0, <vscale x
ret <vscale x 2 x i1> %retval
}
+define <vscale x 24 x i1> @interleave3_nxv24i1(<vscale x 8 x i1> %vec0, <vscale x 8 x i1> %vec1, <vscale x 8 x i1> %vec2) {
+; CHECK-LABEL: interleave3_nxv24i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z2.h, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z1.h, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: st3h { z0.h - z2.h }, p0, [sp]
+; CHECK-NEXT: ldr z0, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z1, [sp]
+; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0
+; CHECK-NEXT: cmpne p2.h, p0/z, z1.h, #0
+; CHECK-NEXT: cmpne p3.h, p0/z, z2.h, #0
+; CHECK-NEXT: uzp1 p0.b, p2.b, p1.b
+; CHECK-NEXT: uzp1 p1.b, p3.b, p0.b
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call <vscale x 24 x i1> @llvm.vector.interleave3.nxv24i1(<vscale x 8 x i1> %vec0, <vscale x 8 x i1> %vec1, <vscale x 8 x i1> %vec2)
+ ret <vscale x 24 x i1> %retval
+}
+
+define <vscale x 48 x i1> @interleave3_nxv48i1(<vscale x 16 x i1> %vec0, <vscale x 16 x i1> %vec1, <vscale x 16 x i1> %vec2) {
+; CHECK-LABEL: interleave3_nxv48i1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z2.b, p2/z, #1 // =0x1
+; CHECK-NEXT: mov z1.b, p1/z, #1 // =0x1
+; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p2.b
+; CHECK-NEXT: st3b { z0.b - z2.b }, p2, [sp]
+; CHECK-NEXT: ldr z0, [sp]
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z2, [sp, #2, mul vl]
+; CHECK-NEXT: cmpne p0.b, p2/z, z0.b, #0
+; CHECK-NEXT: cmpne p1.b, p2/z, z1.b, #0
+; CHECK-NEXT: cmpne p2.b, p2/z, z2.b, #0
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %retval = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %vec0, <vscale x 16 x i1> %vec1, <vscale x 16 x i1> %vec2)
+ ret <vscale x 48 x i1> %retval
+}
+
; Split illegal type size
define <vscale x 16 x i32> @interleave2_nxv16i32(<vscale x 8 x i32> %vec0, <vscale x 8 x i32> %vec1) {
More information about the llvm-commits
mailing list