[llvm] [AArch64] Lower fixed-length interleaved stores with SVE (PR #213692)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 07:58:37 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-aarch64
Author: Kamlesh Kumar (kamleshbhalui)
<details>
<summary>Changes</summary>
Allow uses of SVE instruction for fixed length vector interleave intrinsic that can not be lowered through NEON.
---
Full diff: https://github.com/llvm/llvm-project/pull/213692.diff
2 Files Affected:
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (+39-6)
- (added) llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll (+153)
``````````diff
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 51be0e66b19b0..97e166b782168 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -3715,6 +3715,8 @@ static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
static SDValue convertFixedMaskToScalableVector(SDValue Mask,
SelectionDAG &DAG);
static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT);
+static SDValue getPredicateForFixedLengthVector(SelectionDAG &DAG, SDLoc &DL,
+ EVT VT);
static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
EVT VT);
static SDValue getSVEPredicateBitCast(EVT VT, SDValue Op, SelectionDAG &DAG);
@@ -27775,12 +27777,8 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
return SDValue();
bool IsScalable = SubVecTy.isScalableVector();
unsigned SubBits = SubVecTy.getSizeInBits().getKnownMinValue();
- if (IsScalable) {
- if (SubBits != 128)
- return SDValue();
- } else if (SubBits != 64 && SubBits != 128) {
+ if (IsScalable && SubBits != 128)
return SDValue();
- }
auto *MemN = cast<MemSDNode>(N);
if (IsScalable) {
@@ -27805,8 +27803,12 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
DAG.getVTList(MVT::Other), Ops,
MemN->getMemoryVT(), MemN->getMemOperand());
- } else {
+ }
+ const AArch64Subtarget &Subtarget = DAG.getSubtarget<AArch64Subtarget>();
+ // Fixed length vector using NEON
+ if (!IsMasked && (SubBits == 64 || SubBits == 128) &&
+ Subtarget.isNeonAvailable()) {
static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::aarch64_neon_st2,
Intrinsic::aarch64_neon_st3,
Intrinsic::aarch64_neon_st4};
@@ -27820,6 +27822,37 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
DAG.getVTList(MVT::Other), Ops,
MemN->getMemoryVT(), MemN->getMemOperand());
}
+
+ // Fixed-length vectors using SVE
+ if (!Subtarget.isSVEorStreamingSVEAvailable())
+ return SDValue();
+
+ EVT ContainerVT = getContainerForFixedLengthVector(DAG, SubVecTy);
+ SDValue Pred;
+ if (IsMasked) {
+ Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts);
+ if (!Pred)
+ return SDValue();
+ EVT MaskVT = SubVecTy.changeTypeToInteger();
+ // Widen the i1 mask
+ if (Pred.getValueType() != MaskVT)
+ Pred = DAG.getNode(ISD::SIGN_EXTEND, DL, MaskVT, Pred);
+ Pred = convertFixedMaskToScalableVector(Pred, DAG);
+ } else {
+ Pred = getPredicateForFixedLengthVector(DAG, DL, SubVecTy);
+ }
+
+ static constexpr Intrinsic::ID SVEStores[] = {Intrinsic::aarch64_sve_st2,
+ Intrinsic::aarch64_sve_st3,
+ Intrinsic::aarch64_sve_st4};
+ SmallVector<SDValue, 8> Ops;
+ Ops.append({Chain, DAG.getConstant(SVEStores[NumParts - 2], DL, MVT::i32)});
+ for (SDValue V : ValueInterleaveOps)
+ Ops.push_back(convertToScalableVector(DAG, ContainerVT, V));
+ Ops.append({Pred, BasePtr});
+ return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
+ DAG.getVTList(MVT::Other), Ops,
+ MemN->getMemoryVT(), MemN->getMemOperand());
}
static SDValue performMSTORECombine(SDNode *N,
diff --git a/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
new file mode 100644
index 0000000000000..e3ef467284269
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fixed-length-sve-interleave.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IAENABLED
+; RUN: llc -mtriple=aarch64-linux-gnu --lower-interleaved-accesses=false < %s | FileCheck %s --check-prefixes=CHECK,CHECK-IADISABLED
+
+define void @store_factor2_intrinsic(ptr %ptr, <16 x i16> %v0, <16 x i16> %v1) #0 {
+; CHECK-LABEL: store_factor2_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.h, vl8
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q2 killed $q2 def $z2
+; CHECK-NEXT: adrp x8, .LCPI0_0
+; CHECK-NEXT: add x8, x8, :lo12:.LCPI0_0
+; CHECK-NEXT: ldr z4, [x8]
+; CHECK-NEXT: splice z1.h, p0, z1.h, z3.h
+; CHECK-NEXT: splice z0.h, p0, z0.h, z2.h
+; CHECK-NEXT: tbl z1.h, { z1.h }, z4.h
+; CHECK-NEXT: tbl z0.h, { z0.h }, z4.h
+; CHECK-NEXT: str z1, [x0, #1, mul vl]
+; CHECK-NEXT: str z0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <32 x i16> @llvm.vector.interleave2.v32i16(<16 x i16> %v0, <16 x i16> %v1)
+ store <32 x i16> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2) #0 {
+; CHECK-LABEL: store_factor3_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ store <24 x i32> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3) #0 {
+; CHECK-LABEL: store_factor4_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ store <16 x i64> %interleaved.vec, ptr %ptr, align 4
+ ret void
+}
+
+define void @masked_store_factor3_intrinsic(ptr %ptr, <8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2, <8 x i1> %mask) #0 {
+; CHECK-LABEL: masked_store_factor3_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: // kill: def $d6 killed $d6 def $z6
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: uunpklo z0.h, z6.b
+; CHECK-NEXT: splice z18.s, p0, z18.s, z5.s
+; CHECK-NEXT: splice z17.s, p0, z17.s, z3.s
+; CHECK-NEXT: splice z16.s, p0, z16.s, z1.s
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: lsl z0.s, z0.s, #31
+; CHECK-NEXT: asr z0.s, z0.s, #31
+; CHECK-NEXT: cmpne p0.s, p1/z, z0.s, #0
+; CHECK-NEXT: st3w { z16.s - z18.s }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <24 x i32> @llvm.vector.interleave3.v24i32(<8 x i32> %v0, <8 x i32> %v1, <8 x i32> %v2)
+ %interleaved.mask = call <24 x i1> @llvm.vector.interleave3.v24i1(<8 x i1> %mask, <8 x i1> %mask, <8 x i1> %mask)
+ call void @llvm.masked.store.v24i32.p0(<24 x i32> %interleaved.vec, ptr %ptr, i32 4, <24 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor4_intrinsic(ptr %ptr, <4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3, <4 x i1> %mask) #0 {
+; CHECK-LABEL: masked_store_factor4_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov v19.16b, v6.16b
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: // kill: def $q7 killed $q7 def $z7
+; CHECK-NEXT: // kill: def $q5 killed $q5 def $z5
+; CHECK-NEXT: // kill: def $q3 killed $q3 def $z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT: mov v18.16b, v4.16b
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: mov v17.16b, v2.16b
+; CHECK-NEXT: ldr d2, [sp]
+; CHECK-NEXT: mov v16.16b, v0.16b
+; CHECK-NEXT: uunpklo z2.s, z2.h
+; CHECK-NEXT: splice z19.d, p0, z19.d, z7.d
+; CHECK-NEXT: splice z18.d, p0, z18.d, z5.d
+; CHECK-NEXT: splice z17.d, p0, z17.d, z3.d
+; CHECK-NEXT: splice z16.d, p0, z16.d, z1.d
+; CHECK-NEXT: uunpklo z0.d, z2.s
+; CHECK-NEXT: lsl z0.d, z0.d, #63
+; CHECK-NEXT: asr z0.d, z0.d, #63
+; CHECK-NEXT: cmpne p0.d, p1/z, z0.d, #0
+; CHECK-NEXT: st4d { z16.d - z19.d }, p0, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <16 x i64> @llvm.vector.interleave4.v16i64(<4 x i64> %v0, <4 x i64> %v1, <4 x i64> %v2, <4 x i64> %v3)
+ %interleaved.mask = call <16 x i1> @llvm.vector.interleave4.v16i1(<4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v16i64.p0(<16 x i64> %interleaved.vec, ptr %ptr, i32 8, <16 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @masked_store_factor3_128bit_intrinsic(ptr %ptr, <4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2, <4 x i1> %mask) #0 {
+; CHECK-LABEL: masked_store_factor3_128bit_intrinsic:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1_z2 def $z0_z1_z2
+; CHECK-NEXT: shl v3.4s, v3.4s, #31
+; CHECK-NEXT: cmpne p1.s, p0/z, z3.s, #0
+; CHECK-NEXT: st3w { z0.s - z2.s }, p1, [x0]
+; CHECK-NEXT: ret
+ %interleaved.vec = call <12 x i32> @llvm.vector.interleave3.v12i32(<4 x i32> %v0, <4 x i32> %v1, <4 x i32> %v2)
+ %interleaved.mask = call <12 x i1> @llvm.vector.interleave3.v12i1(<4 x i1> %mask, <4 x i1> %mask, <4 x i1> %mask)
+ call void @llvm.masked.store.v12i32.p0(<12 x i32> %interleaved.vec, ptr %ptr, i32 4, <12 x i1> %interleaved.mask)
+ ret void
+}
+
+attributes #0 = { vscale_range(2,2) "target-features"="+sve" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-IADISABLED: {{.*}}
+; CHECK-IAENABLED: {{.*}}
``````````
</details>
https://github.com/llvm/llvm-project/pull/213692
More information about the llvm-commits
mailing list