[llvm] [AArch64][SVE] Allow scalable factor-3 interleaved accesses (PR #200424)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 9 07:52:30 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/200424
>From d6a0f2b93dd52c71a33c5278b9fc7c8aaafa19b1 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Fri, 3 Jul 2026 14:47:56 +0000
Subject: [PATCH] Enable factor-3 scalable interleaved memory costs
Allow scalable interleaved memory operations with factor 3 in the
AArch64 TTI cost model.
SelectionDAG can lower explicit scalable vector.interleave3 and
vector.deinterleave3 patterns for legal packed SVE vector types. For
vector.interleave3 values declared in seperate blocks must be
deinterleaved first before being used.
---
.../Target/AArch64/AArch64ISelLowering.cpp | 54 ++++++++++++++-----
.../AArch64/AArch64TargetTransformInfo.cpp | 4 +-
.../scalable_masked_deinterleaved_loads.ll | 11 ++++
.../scalable_masked_interleaved_stores.ll | 14 +++++
4 files changed, 67 insertions(+), 16 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ea46428dfa828..659e7ad5dfe70 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27437,7 +27437,7 @@ static SDValue performInterleavedMaskedStoreCombine(
return SDValue();
unsigned NumParts = ValueInterleaveOps.size();
- if (NumParts != 2 && NumParts != 4)
+ if (NumParts != 2 && NumParts != 3 && NumParts != 4)
return SDValue();
// At the moment we're unlikely to see a fixed-width vector interleave as
@@ -27454,8 +27454,18 @@ static SDValue performInterleavedMaskedStoreCombine(
if (!NarrowMask)
return SDValue();
- const Intrinsic::ID IID =
- NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4;
+ Intrinsic::ID IID;
+ switch (NumParts) {
+ case 2:
+ IID = Intrinsic::aarch64_sve_st2;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_sve_st3;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_sve_st4;
+ break;
+ }
SmallVector<SDValue, 8> NewStOps;
NewStOps.append({MST->getChain(), DAG.getConstant(IID, DL, MVT::i32)});
NewStOps.append(ValueInterleaveOps);
@@ -30343,7 +30353,7 @@ static SDValue performVectorDeinterleaveCombine(
return SDValue();
unsigned NumParts = N->getNumOperands();
- if (NumParts != 2 && NumParts != 4)
+ if (NumParts != 2 && NumParts != 3 && NumParts != 4)
return SDValue();
EVT SubVecTy = N->getValueType(0);
@@ -30389,19 +30399,37 @@ static SDValue performVectorDeinterleaveCombine(
if (!NarrowMask)
return SDValue();
- const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret
- : Intrinsic::aarch64_sve_ld4_sret;
+ Intrinsic::ID IID;
+ switch(NumParts) {
+ case 2:
+ IID = Intrinsic::aarch64_sve_ld2_sret;
+ break;
+ case 3:
+ IID = Intrinsic::aarch64_sve_ld3_sret;
+ break;
+ case 4:
+ IID = Intrinsic::aarch64_sve_ld4_sret;
+ break;
+ }
SDValue NewLdOps[] = {MaskedLoad->getChain(),
DAG.getConstant(IID, DL, MVT::i32), NarrowMask,
MaskedLoad->getBasePtr()};
SDValue Res;
- if (NumParts == 2)
- Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
- {SubVecTy, SubVecTy, MVT::Other}, NewLdOps);
- else
- Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
- {SubVecTy, SubVecTy, SubVecTy, SubVecTy, MVT::Other},
- NewLdOps);
+ switch (NumParts) {
+ case 2:
+ Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
+ {SubVecTy, SubVecTy, MVT::Other}, NewLdOps);
+ break;
+ case 3:
+ Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
+ {SubVecTy, SubVecTy, SubVecTy, MVT::Other}, NewLdOps);
+ break;
+ case 4:
+ Res = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
+ {SubVecTy, SubVecTy, SubVecTy, SubVecTy, MVT::Other},
+ NewLdOps);
+ break;
+ }
// We can now generate a structured load!
SmallVector<SDValue, 4> ResOps(NumParts);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index caca744fd4303..5332be073c124 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5387,9 +5387,7 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
// Scalable VFs will emit vector.[de]interleave intrinsics, and currently we
// only have lowering for power-of-2 factors.
- // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in
- // InterleavedAccessPass for ld3/st3
- if (VecTy->isScalableTy() && !isPowerOf2_32(Factor))
+ if (VecTy->isScalableTy() && Factor > TLI->getMaxSupportedInterleaveFactor())
return InstructionCost::getInvalid();
// Vectorization for masked interleaved accesses is only enabled for scalable
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
index ee2d482d9ffb5..a74cee570f884 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll
@@ -45,6 +45,17 @@ define { <vscale x 2 x double>, <vscale x 2 x double> } @foo_ld2_nxv2f64(<vscale
ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleaved.vec
}
+define { <vscale x 16 x i8 >,<vscale x 16 x i8 >,<vscale x 16 x i8 > } @foo_ld3_nx48i8(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: foo_ld3_nx48i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %wide.masked.vec = call <vscale x 48 x i8> @llvm.masked.load.nxv48i8(ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask, <vscale x 48 x i8> poison)
+ %deinterleaved.vec = call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.vector.deinterleave3.nxv48i8(<vscale x 48 x i8> %wide.masked.vec)
+ ret { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } %deinterleaved.vec
+}
+
define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld4_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) {
; CHECK-LABEL: foo_ld4_nxv16i8:
; CHECK: // %bb.0:
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
index d3cd9bf08cc0a..0eaa3963ff595 100644
--- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll
@@ -53,6 +53,20 @@ define void @foo_st2_nxv2i64(<vscale x 2 x i1> %mask, <vscale x 2 x i64> %val1,
ret void
}
+define void @foo_st3_nxv48i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale x 16 x i8> %v2, ptr %p) {
+; CHECK-LABEL: foo_st3_nxv48i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: st3b { z0.b - z2.b }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %interleaved.value = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> %v0, <vscale x 16 x i8> %v1, <vscale x 16 x i8> %v2)
+ call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> %interleaved.value, ptr %p, i32 1, <vscale x 48 x i1>%interleaved.mask)
+ ret void
+}
+
define void @foo_st4_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, <vscale x 16 x i8> %val4, ptr %p) {
; CHECK-LABEL: foo_st4_nxv16i8:
; CHECK: // %bb.0:
More information about the llvm-commits
mailing list