[llvm] [AArch64][SVE] Enable factor-3 scalable interleave operations (PR #200424)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 03:27:37 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/200424
>From 3d0879e3eb0b275ba65828e6810e32d52be64664 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Sun, 2 Aug 2026 22:48:33 +0000
Subject: [PATCH] [AArch64][ISel] Enable factor-3 scalable interleave
operations
Enable factor-3 scalable vector interleave and deinterleave lowering for SVE,
including the corresponding interleaved load/store combines.
Factor-3 operations are lowered through the existing LD3/ST3-based path, which
operates on packed data vectors.
Unlike factors 2 and 4, SVE has no ZIP/UZP instruction sequence specifically
ror factor-3 operations. For LD3/ST3 predicate operands must be promoted
into Z-register data vectors before lowering, then convert the results
back to predicates by comparing against zero. This avoids invalid
predicate uses of getSVESafeBitCast.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 32 ++++++++---
.../scalable_masked_deinterleaved_loads.ll | 44 +++++++++++++++
.../scalable_masked_interleaved_stores.ll | 56 +++++++++++++++++++
3 files changed, 124 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 51be0e66b19b0..c6f163c2d0a16 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27784,8 +27784,6 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
auto *MemN = cast<MemSDNode>(N);
if (IsScalable) {
- if (NumParts == 3)
- return SDValue();
SDValue Pred;
if (IsMasked) {
Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts);
@@ -27796,8 +27794,18 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
Pred = DAG.getConstant(1, DL, PredVT);
}
- const Intrinsic::ID IID =
- NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4;
+ Intrinsic::ID IID;
+ switch (NumParts) {
+ case 2:
+ IID = Intrinsic::aarch64_sve_st2;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_sve_st3;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_sve_st4;
+ break;
+ }
SmallVector<SDValue, 8> Ops;
Ops.append({Chain, DAG.getConstant(IID, DL, MVT::i32)});
Ops.append(ValueInterleaveOps);
@@ -30848,8 +30856,6 @@ static SDValue performVectorDeinterleaveCombine(
SDValue Res;
MemSDNode *MemNode = dyn_cast<MemSDNode>(WideVec);
if (IsScalable) {
- if (NumParts == 3)
- return SDValue();
SDValue Chain, BasePtr, Pred;
if (auto *MaskedLoad = dyn_cast<MaskedLoadSDNode>(WideVec)) {
// Bail out if the masked load has an unexpected number of uses, since we
@@ -30883,8 +30889,18 @@ static SDValue performVectorDeinterleaveCombine(
BasePtr = Load->getBasePtr();
}
- const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret
- : Intrinsic::aarch64_sve_ld4_sret;
+ Intrinsic::ID IID;
+ switch (NumParts) {
+ case 2:
+ IID = Intrinsic::aarch64_sve_ld2_sret;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_sve_ld3_sret;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_sve_ld4_sret;
+ break;
+ }
SDValue NewLdOps[] = {Chain, DAG.getConstant(IID, DL, MVT::i32), Pred,
BasePtr};
Res = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, ResVTList,
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index ee2d482d9ffb5..f5129c7c57826 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -45,6 +45,50 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @foo_ld2_nxv2f64(<vscale
ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleaved.vec
}
+define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld3_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld3_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %wide.masked.vec = call <vscale x 48 x i8> @llvm.masked.load.nxv48i8(ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask, <vscale x 48 x i8> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave3.nxv48i8(<vscale x 48 x i8> %wide.masked.vec)
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.vec
+}
+
+define { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @foo_ld3_nxv8i16(<vscale x 8 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld3_nxv8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 24 x i1> @llvm.vector.interleave3.nxv24i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %wide.masked.vec = call <vscale x 24 x i16> @llvm.masked.load.nxv24i16(ptr %p, i32 1, <vscale x 24 x i1> %interleaved.mask, <vscale x 24 x i16> poison)
+ %deinterleaved.vec = call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.vector.deinterleave3.nxv24i16(<vscale x 24 x i16> %wide.masked.vec)
+ ret { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } %deinterleaved.vec
+}
+
+define { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @foo_ld3_nxv4i32(<vscale x 4 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld3_nxv4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 12 x i1> @llvm.vector.interleave3.nxv12i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %wide.masked.vec = call <vscale x 12 x i32> @llvm.masked.load.nxv12i32(ptr %p, i32 1, <vscale x 12 x i1> %interleaved.mask, <vscale x 12 x i32> poison)
+ %deinterleaved.vec = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> %wide.masked.vec)
+ ret { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } %deinterleaved.vec
+}
+
+define { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @foo_ld3_nxv2i64(<vscale x 2 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld3_nxv2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 6 x i1> @llvm.vector.interleave3.nxv6i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %wide.masked.vec = call <vscale x 6 x i64> @llvm.masked.load.nxv6i64(ptr %p, i32 1, <vscale x 6 x i1> %interleaved.mask, <vscale x 6 x i64> poison)
+ %deinterleaved.vec = call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.vector.deinterleave3.nxv6i64(<vscale x 6 x i64> %wide.masked.vec)
+ ret { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } %deinterleaved.vec
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld4_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld4_nxv16i8:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index fc2f64b756959..766777925e909 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -54,6 +54,62 @@ define void @foo_st2_nxv2i64(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %val1,
ret void
}
+define void @foo_st3_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv16i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %interleaved.value = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3)
+ call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st3_nxv8i16(<vscale x 8 x i1> %mask, <vscale x 8 x i16> %val1, <vscale x 8 x i16> %val2, <vscale x 8 x i16> %val3, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: st3h { z0.h - z2.h }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 24 x i1> @llvm.vector.interleave3.nxv24i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %interleaved.value = call <vscale x 24 x i16> @llvm.vector.interleave3.nxv24i16(<vscale x 8 x i16> %val1, <vscale x 8 x i16> %val2, <vscale x 8 x i16> %val3)
+ call void @llvm.masked.store.nxv24i16.p0(<vscale x 24 x i16> %interleaved.value, ptr %p, i32 1, <vscale x 24 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st3_nxv4i32(<vscale x 4 x i1> %mask, <vscale x 4 x i32> %val1, <vscale x 4 x i32> %val2, <vscale x 4 x i32> %val3, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 12 x i1> @llvm.vector.interleave3.nxv12i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %interleaved.value = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> %val1, <vscale x 4 x i32> %val2, <vscale x 4 x i32> %val3)
+ call void @llvm.masked.store.nxv12i32.p0(<vscale x 12 x i32> %interleaved.value, ptr %p, i32 1, <vscale x 12 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @foo_st3_nxv2i64(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %val1, <vscale x 2 x i64> %val2, <vscale x 2 x i64> %val3, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv2i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 6 x i1> @llvm.vector.interleave3.nxv6i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %interleaved.value = call <vscale x 6 x i64> @llvm.vector.interleave3.nxv6i64(<vscale x 2 x i64> %val1, <vscale x 2 x i64> %val2, <vscale x 2 x i64> %val3)
+ call void @llvm.masked.store.nxv6i64.p0(<vscale x 6 x i64> %interleaved.value, ptr %p, i32 1, <vscale x 6 x i1> %interleaved.mask)
+ ret void
+}
+
define void @foo_st4_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, <vscale x 16 x i8> %val4, ptr %p) {
; CHECK-LABEL: foo_st4_nxv16i8:
; CHECK: // %bb.0:
More information about the llvm-commits
mailing list