[llvm] [AArch64][Vectorizer] Enable scalable factor-3 interleaving (PR #217370)
Harry Ramsey via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 26 08:11:29 PDT 2026
https://github.com/Harry-Ramsey updated https://github.com/llvm/llvm-project/pull/217370
>From b6c212c00114ade0522e1958806d0b7fd9a4dcd3 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Tue, 25 Aug 2026 20:04:07 +0000
Subject: [PATCH 1/2] [SelectionDAG] Legalize non-divisible scalable vector
widening
Enable DAGTypeLegalizer::ModifyToType to widen scalable vectors whose
element counts are not integer multiples. Split the source and
destination into their largest common scalable subvector type, extract
the source parts, append poison or zero fill parts, and concatenate
them.
---
.../SelectionDAG/LegalizeVectorTypes.cpp | 26 ++++
.../CodeGen/AArch64/scalable_masked_loads.ll | 118 ++++++++++++++++++
.../CodeGen/AArch64/scalable_masked_stores.ll | 118 ++++++++++++++++++
3 files changed, 262 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/scalable_masked_loads.ll
create mode 100644 llvm/test/CodeGen/AArch64/scalable_masked_stores.ll
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 6292fac464c97..08020577beb72 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -9259,6 +9259,7 @@ SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
if (InEC.hasKnownScalarFactor(WidenEC))
return DAG.getExtractSubvector(dl, NVT, InOp, 0);
+<<<<<<< HEAD
if (NVT.isScalableVector() && InVT.isScalableVector()) {
// Split the input into the largest equal-sized scalable subvectors.
unsigned InNumElts = InVT.getVectorMinNumElements();
@@ -9270,16 +9271,41 @@ SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
SmallVector<SDValue, 16> Ops;
unsigned NumCopiedParts = std::min(InNumElts, NewNumElts) / CommonFactor;
for (unsigned I = 0; I != NumCopiedParts; ++I)
+=======
+ if (InVT.isScalableVector()) {
+ // Split the input and widened vector into the largest equal-sized scalable
+ // subvectors. The common factor lets us preserve the source lanes and pad
+ // with subvectors, avoiding per-lane operations on scalable vectors.
+ // Worst case scenario (GCD == 1) we fall back to single operations.
+ unsigned InNumElts = InVT.getVectorMinNumElements();
+ unsigned NewNumElts = NVT.getVectorMinNumElements();
+ unsigned CommonFactor = std::gcd(InNumElts, NewNumElts);
+ EVT PartVT = EVT::getVectorVT(
+ *DAG.getContext(), NVT.getVectorElementType(),
+ ElementCount::getScalable(CommonFactor));
+
+ SmallVector<SDValue, 16> Ops;
+ unsigned NumSourceParts = InNumElts / CommonFactor;
+ for (unsigned I = 0; I != NumSourceParts; ++I)
+>>>>>>> 65cbf676fe17 ([SelectionDAG] Legalize non-divisible scalable vector widening)
Ops.push_back(
DAG.getExtractSubvector(dl, PartVT, InOp, I * CommonFactor));
unsigned NumResultParts = NewNumElts / CommonFactor;
+<<<<<<< HEAD
if (NumResultParts > NumCopiedParts) {
SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, PartVT)
: DAG.getPOISON(PartVT);
Ops.append(NumResultParts - NumCopiedParts, FillVal);
}
+=======
+ if (NumSourceParts != NumResultParts) {
+ SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, PartVT)
+ : DAG.getPOISON(PartVT);
+ Ops.append(NumResultParts - NumSourceParts, FillVal);
+ }
+>>>>>>> 65cbf676fe17 ([SelectionDAG] Legalize non-divisible scalable vector widening)
return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
}
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_loads.ll
new file mode 100644
index 0000000000000..fe39f711c1eb8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_loads.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define <vscale x 48 x i8> @load_nxv48i8(<vscale x 16 x i1> %mask, ptr %p) {
+; CHECK-LABEL: load_nxv48i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p1.b
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: st3b { z0.b - z2.b }, p1, [sp]
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z2, [sp]
+; CHECK-NEXT: cmpne p0.b, p1/z, z0.b, #0
+; CHECK-NEXT: cmpne p2.b, p1/z, z1.b, #0
+; CHECK-NEXT: cmpne p3.b, p1/z, z2.b, #0
+; CHECK-NEXT: ld1b { z0.b }, p3/z, [x0]
+; CHECK-NEXT: ld1b { z1.b }, p2/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1b { z2.b }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ %wide.masked.vec = call <vscale x 48 x i8> @llvm.masked.load.nxv48i8(ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask, <vscale x 48 x i8> poison)
+ ret <vscale x 48 x i8> %wide.masked.vec
+}
+
+define <vscale x 24 x i16> @load_nxv24i16(<vscale x 8 x i1> %mask, ptr %p) {
+; CHECK-LABEL: load_nxv24i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: st3h { z0.h - z2.h }, p1, [sp]
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z2, [sp]
+; CHECK-NEXT: cmpne p0.h, p1/z, z0.h, #0
+; CHECK-NEXT: cmpne p2.h, p1/z, z1.h, #0
+; CHECK-NEXT: cmpne p3.h, p1/z, z2.h, #0
+; CHECK-NEXT: ld1h { z0.h }, p3/z, [x0]
+; CHECK-NEXT: ld1h { z1.h }, p2/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 24 x i1> @llvm.vector.interleave3.nxv24i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ %wide.masked.vec = call <vscale x 24 x i16> @llvm.masked.load.nxv24i16(ptr %p, i32 1, <vscale x 24 x i1> %interleaved.mask, <vscale x 24 x i16> poison)
+ ret <vscale x 24 x i16> %wide.masked.vec
+}
+
+define <vscale x 12 x i32> @load_nxv12i32(<vscale x 4 x i1> %mask, ptr %p) {
+; CHECK-LABEL: load_nxv12i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: st3w { z0.s - z2.s }, p1, [sp]
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z2, [sp]
+; CHECK-NEXT: cmpne p0.s, p1/z, z0.s, #0
+; CHECK-NEXT: cmpne p2.s, p1/z, z1.s, #0
+; CHECK-NEXT: cmpne p3.s, p1/z, z2.s, #0
+; CHECK-NEXT: ld1w { z0.s }, p3/z, [x0]
+; CHECK-NEXT: ld1w { z1.s }, p2/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1w { z2.s }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 12 x i1> @llvm.vector.interleave3.nxv12i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ %wide.masked.vec = call <vscale x 12 x i32> @llvm.masked.load.nxv12i32(ptr %p, i32 1, <vscale x 12 x i1> %interleaved.mask, <vscale x 12 x i32> poison)
+ ret <vscale x 12 x i32> %wide.masked.vec
+}
+
+define <vscale x 6 x i64> @load_nxv6i64(<vscale x 2 x i1> %mask, ptr %p) {
+; CHECK-LABEL: load_nxv6i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: st3d { z0.d - z2.d }, p1, [sp]
+; CHECK-NEXT: ldr z0, [sp, #2, mul vl]
+; CHECK-NEXT: ldr z1, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z2, [sp]
+; CHECK-NEXT: cmpne p0.d, p1/z, z0.d, #0
+; CHECK-NEXT: cmpne p2.d, p1/z, z1.d, #0
+; CHECK-NEXT: cmpne p3.d, p1/z, z2.d, #0
+; CHECK-NEXT: ld1d { z0.d }, p3/z, [x0]
+; CHECK-NEXT: ld1d { z1.d }, p2/z, [x0, #1, mul vl]
+; CHECK-NEXT: ld1d { z2.d }, p0/z, [x0, #2, mul vl]
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 6 x i1> @llvm.vector.interleave3.nxv6i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ %wide.masked.vec = call <vscale x 6 x i64> @llvm.masked.load.nxv6i64(ptr %p, i32 1, <vscale x 6 x i1> %interleaved.mask, <vscale x 6 x i64> poison)
+ ret <vscale x 6 x i64> %wide.masked.vec
+}
diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_stores.ll
new file mode 100644
index 0000000000000..74656a5c61ba7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/scalable_masked_stores.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+define void @store_nxv48i8(<vscale x 16 x i1> %mask, <vscale x 48 x i8> %value, ptr %p) {
+; CHECK-LABEL: store_nxv48i8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z3.b, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p1.b
+; CHECK-NEXT: mov z4.d, z3.d
+; CHECK-NEXT: mov z5.d, z3.d
+; CHECK-NEXT: st3b { z3.b - z5.b }, p1, [sp]
+; CHECK-NEXT: ldr z3, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z4, [sp]
+; CHECK-NEXT: ldr z5, [sp, #2, mul vl]
+; CHECK-NEXT: cmpne p0.b, p1/z, z3.b, #0
+; CHECK-NEXT: cmpne p2.b, p1/z, z4.b, #0
+; CHECK-NEXT: cmpne p3.b, p1/z, z5.b, #0
+; CHECK-NEXT: st1b { z1.b }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: st1b { z0.b }, p2, [x0]
+; CHECK-NEXT: st1b { z2.b }, p3, [x0, #2, mul vl]
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 48 x i1> @llvm.vector.interleave3.nxv48i1(<vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask, <vscale x 16 x i1> %mask)
+ call void @llvm.masked.store.nxv48i8.p0(<vscale x 48 x i8> %value, ptr %p, i32 1, <vscale x 48 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @store_nxv24i16(<vscale x 8 x i1> %mask, <vscale x 24 x i16> %value, ptr %p) {
+; CHECK-LABEL: store_nxv24i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z3.h, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: mov z4.d, z3.d
+; CHECK-NEXT: mov z5.d, z3.d
+; CHECK-NEXT: st3h { z3.h - z5.h }, p1, [sp]
+; CHECK-NEXT: ldr z3, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z4, [sp]
+; CHECK-NEXT: ldr z5, [sp, #2, mul vl]
+; CHECK-NEXT: cmpne p0.h, p1/z, z3.h, #0
+; CHECK-NEXT: cmpne p2.h, p1/z, z4.h, #0
+; CHECK-NEXT: cmpne p3.h, p1/z, z5.h, #0
+; CHECK-NEXT: st1h { z1.h }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: st1h { z0.h }, p2, [x0]
+; CHECK-NEXT: st1h { z2.h }, p3, [x0, #2, mul vl]
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 24 x i1> @llvm.vector.interleave3.nxv24i1(<vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask, <vscale x 8 x i1> %mask)
+ call void @llvm.masked.store.nxv24i16.p0(<vscale x 24 x i16> %value, ptr %p, i32 2, <vscale x 24 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @store_nxv12i32(<vscale x 4 x i1> %mask, <vscale x 12 x i32> %value, ptr %p) {
+; CHECK-LABEL: store_nxv12i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z3.s, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: mov z4.d, z3.d
+; CHECK-NEXT: mov z5.d, z3.d
+; CHECK-NEXT: st3w { z3.s - z5.s }, p1, [sp]
+; CHECK-NEXT: ldr z3, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z4, [sp]
+; CHECK-NEXT: ldr z5, [sp, #2, mul vl]
+; CHECK-NEXT: cmpne p0.s, p1/z, z3.s, #0
+; CHECK-NEXT: cmpne p2.s, p1/z, z4.s, #0
+; CHECK-NEXT: cmpne p3.s, p1/z, z5.s, #0
+; CHECK-NEXT: st1w { z1.s }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: st1w { z0.s }, p2, [x0]
+; CHECK-NEXT: st1w { z2.s }, p3, [x0, #2, mul vl]
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 12 x i1> @llvm.vector.interleave3.nxv12i1(<vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask, <vscale x 4 x i1> %mask)
+ call void @llvm.masked.store.nxv12i32.p0(<vscale x 12 x i32> %value, ptr %p, i32 4, <vscale x 12 x i1> %interleaved.mask)
+ ret void
+}
+
+define void @store_nxv6i64(<vscale x 2 x i1> %mask, <vscale x 6 x i64> %value, ptr %p) {
+; CHECK-LABEL: store_nxv6i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: addvl sp, sp, #-3
+; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG
+; CHECK-NEXT: .cfi_offset w29, -16
+; CHECK-NEXT: mov z3.d, p0/z, #1 // =0x1
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: mov z4.d, z3.d
+; CHECK-NEXT: mov z5.d, z3.d
+; CHECK-NEXT: st3d { z3.d - z5.d }, p1, [sp]
+; CHECK-NEXT: ldr z3, [sp, #1, mul vl]
+; CHECK-NEXT: ldr z4, [sp]
+; CHECK-NEXT: ldr z5, [sp, #2, mul vl]
+; CHECK-NEXT: cmpne p0.d, p1/z, z3.d, #0
+; CHECK-NEXT: cmpne p2.d, p1/z, z4.d, #0
+; CHECK-NEXT: cmpne p3.d, p1/z, z5.d, #0
+; CHECK-NEXT: st1d { z1.d }, p0, [x0, #1, mul vl]
+; CHECK-NEXT: st1d { z0.d }, p2, [x0]
+; CHECK-NEXT: st1d { z2.d }, p3, [x0, #2, mul vl]
+; CHECK-NEXT: addvl sp, sp, #3
+; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
+ %interleaved.mask = call <vscale x 6 x i1> @llvm.vector.interleave3.nxv6i1(<vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask, <vscale x 2 x i1> %mask)
+ call void @llvm.masked.store.nxv6i64.p0(<vscale x 6 x i64> %value, ptr %p, i32 8, <vscale x 6 x i1> %interleaved.mask)
+ ret void
+}
>From 9482504887bc82ed851a1780a43294b0fbce3f43 Mon Sep 17 00:00:00 2001
From: Harry Ramsey <harry.ramsey at arm.com>
Date: Wed, 19 Aug 2026 15:13:15 +0000
Subject: [PATCH 2/2] [AArch64][Vectorizer] Enable scalable factor-3
interleaving
AArch64 can lower scalable `llvm.vector.interleave3` and
`llvm.vector.deinterleave3` intrinsics. Allow the vectorizer to cost these
operations when the interleave factor is supported.
Factor-3 lowering requires each input vector to have a known minimum size of
384 bits, so retain an invalid cost for smaller scalable vector types.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 16 +-
.../AArch64/force-target-instruction-cost.ll | 56 ++++--
.../LoopVectorize/AArch64/interleaved_cost.ll | 177 ++++++++++++++++++
.../AArch64/sve-interleave-low-vf-cost.ll | 85 ++++++++-
.../AArch64/sve-interleaved-accesses.ll | 41 ++--
5 files changed, 324 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 8f0bbc0408d97..d7e700212b6e1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5737,12 +5737,16 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
if (VecTy->isScalableTy() && !ST->hasSVE())
return InstructionCost::getInvalid();
- // Scalable VFs will emit vector.[de]interleave intrinsics, and currently we
- // only have lowering for power-of-2 factors.
- // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in
- // InterleavedAccessPass for ld3/st3
- if (VecTy->isScalableTy() && !isPowerOf2_32(Factor))
- return InstructionCost::getInvalid();
+ // Scalable VFs emit vector.[de]interleave intrinsics, for which the target
+ // supports factors up to the maximum supported interleave factor.
+ if (VecTy->isScalableTy()) {
+ if (Factor > TLI->getMaxSupportedInterleaveFactor())
+ return InstructionCost::getInvalid();
+
+ if (Factor == 3 &&
+ DL.getTypeSizeInBits(VecTy).getKnownMinValue() != (3 * 128))
+ return InstructionCost::getInvalid();
+ }
// Vectorization for masked interleaved accesses is only enabled for scalable
// VF.
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 4bc2adc5af8fc..37c86b6dc0607 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -439,27 +439,39 @@ define void @interleave_group(ptr %dst) #1 {
; COST1-NEXT: [[ITER_CHECK:.*:]]
; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST1-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; COST1-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 5
+; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP25]]
+; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST1: [[VECTOR_PH]]:
+; COST1-NEXT: [[TMP26:%.*]] = shl nuw i64 [[TMP24]], 4
+; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP25]]
+; COST1-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 16
+; COST1-NEXT: [[TMP28:%.*]] = add i64 [[TMP26]], 0
+; COST1-NEXT: [[TMP5:%.*]] = mul i64 [[TMP28]], 1
+; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[TMP5]]
; COST1-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 3
; COST1-NEXT: [[TMP2:%.*]] = mul i64 [[TMP0]], 3
; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; COST1-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1
-; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 1
+; COST1-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC1]], ptr [[TMP4]], align 1
+; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP25]]
+; COST1-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST1-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
-; COST1-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST1: [[VEC_EPILOG_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST1: [[VEC_EPILOG_PH]]:
-; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST1-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
@@ -509,23 +521,31 @@ define void @interleave_group(ptr %dst) #1 {
; COST10-NEXT: [[ITER_CHECK:.*:]]
; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST10: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST10-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; COST10-NEXT: [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP22]]
+; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST10: [[VECTOR_PH]]:
+; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP22]]
+; COST10-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST10-NEXT: br label %[[VECTOR_BODY:.*]]
; COST10: [[VECTOR_BODY]]:
; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST10-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3
; COST10-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
-; COST10-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1
-; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; COST10-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST10-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST10-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST10-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 1
+; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP22]]
+; COST10-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST10-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST10: [[MIDDLE_BLOCK]]:
-; COST10-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST10-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST10: [[VEC_EPILOG_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST10-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST10-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST10: [[VEC_EPILOG_PH]]:
-; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST10-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
index 7ce07a82e6465..b6639ee3d813f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/interleaved_cost.ll
@@ -130,6 +130,183 @@ for.end:
ret void
}
+%i8.3 = type {i8, i8, i8}
+define void @i8_factor_3(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i8_factor_3'
+; VF_4-LABEL: 'i8_factor_3'
+; VF_8-LABEL: 'i8_factor_3'
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_16-LABEL: 'i8_factor_3'
+; VF_16: Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 3 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 0
+ %tmp1 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 1
+ %tmp2 = getelementptr inbounds %i8.3, ptr %data, i64 %i, i32 2
+ %tmp3 = load i8, ptr %tmp0, align 1
+ %tmp4 = load i8, ptr %tmp1, align 1
+ %tmp5 = load i8, ptr %tmp2, align 1
+ store i8 %tmp3, ptr %tmp0, align 1
+ store i8 %tmp4, ptr %tmp1, align 1
+ store i8 %tmp5, ptr %tmp2, align 1
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+%i16.3 = type {i16, i16, i16}
+define void @i16_factor_3(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i16_factor_3'
+; VF_4-LABEL: 'i16_factor_3'
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_8-LABEL: 'i16_factor_3'
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 3 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_16-LABEL: 'i16_factor_3'
+; VF_16: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 6 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 0
+ %tmp1 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 1
+ %tmp2 = getelementptr inbounds %i16.3, ptr %data, i64 %i, i32 2
+ %tmp3 = load i16, ptr %tmp0, align 2
+ %tmp4 = load i16, ptr %tmp1, align 2
+ %tmp5 = load i16, ptr %tmp2, align 2
+ store i16 %tmp3, ptr %tmp0, align 2
+ store i16 %tmp4, ptr %tmp1, align 2
+ store i16 %tmp5, ptr %tmp2, align 2
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+%i32.3 = type {i32, i32, i32}
+define void @i32_factor_3(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i32_factor_3'
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_4-LABEL: 'i32_factor_3'
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 3 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_8-LABEL: 'i32_factor_3'
+; VF_8: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 6 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_16-LABEL: 'i32_factor_3'
+; VF_16: Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 12 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 0
+ %tmp1 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 1
+ %tmp2 = getelementptr inbounds %i32.3, ptr %data, i64 %i, i32 2
+ %tmp3 = load i32, ptr %tmp0, align 4
+ %tmp4 = load i32, ptr %tmp1, align 4
+ %tmp5 = load i32, ptr %tmp2, align 4
+ store i32 %tmp3, ptr %tmp0, align 4
+ store i32 %tmp4, ptr %tmp1, align 4
+ store i32 %tmp5, ptr %tmp2, align 4
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
+%i64.3 = type {i64, i64, i64}
+define void @i64_factor_3(ptr %data, i64 %n) {
+; VF_2-LABEL: 'i64_factor_3'
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_2: Cost of 3 for VF 2: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_4-LABEL: 'i64_factor_3'
+; VF_4: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_4: Cost of 6 for VF 4: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_8-LABEL: 'i64_factor_3'
+; VF_8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_8: Cost of 12 for VF 8: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+; VF_16-LABEL: 'i64_factor_3'
+; VF_16: Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+; VF_16: Cost of 24 for VF 16: INTERLEAVE-GROUP with factor 3, ir<%tmp0>
+;
+entry:
+ br label %for.body
+
+for.body:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.body ]
+ %tmp0 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 0
+ %tmp1 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 1
+ %tmp2 = getelementptr inbounds %i64.3, ptr %data, i64 %i, i32 2
+ %tmp3 = load i64, ptr %tmp0, align 8
+ %tmp4 = load i64, ptr %tmp1, align 8
+ %tmp5 = load i64, ptr %tmp2, align 8
+ store i64 %tmp3, ptr %tmp0, align 8
+ store i64 %tmp4, ptr %tmp1, align 8
+ store i64 %tmp5, ptr %tmp2, align 8
+ %i.next = add nuw nsw i64 %i, 1
+ %cond = icmp slt i64 %i.next, %n
+ br i1 %cond, label %for.body, label %for.end
+
+for.end:
+ ret void
+}
+
%i64.8 = type {i64, i64, i64, i64, i64, i64, i64, i64}
define void @i64_factor_8(ptr %data, i64 %n) {
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
index 2bd9a42404a5b..0ef07336d1113 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
@@ -70,6 +70,48 @@ exit:
ret void
}
+define void @deinterleave3_nxv3i16_load(ptr noalias readonly %src, ptr noalias %out, i64 %n) #0 {
+; CHECK-LABEL: LV: Checking a loop in 'deinterleave3_nxv3i16_load'
+; CHECK: Cost of 3 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%ptr.b>
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sum.b = phi double[ 0.0, %entry ], [ %add.b, %loop ]
+ %sum.g = phi double[ 0.0, %entry ], [ %add.g, %loop ]
+ %sum.r = phi double[ 0.0, %entry ], [ %add.r, %loop ]
+
+ %ptr.b = getelementptr inbounds { i16, i16, i16 }, ptr %src, i64 %iv, i32 0
+ %load.b = load i16, ptr %ptr.b, align 2
+
+ %ptr.g = getelementptr inbounds { i16, i16, i16 }, ptr %src, i64 %iv, i32 1
+ %load.g = load i16, ptr %ptr.g, align 2
+
+ %ptr.r = getelementptr inbounds { i16, i16, i16 }, ptr %src, i64 %iv, i32 2
+ %load.r = load i16, ptr %ptr.r, align 2
+
+ %ext.b = uitofp i16 %load.b to double
+ %ext.g = uitofp i16 %load.g to double
+ %ext.r = uitofp i16 %load.r to double
+
+ %add.b = fadd double %sum.b, %ext.b
+ %add.g = fadd double %sum.g, %ext.g
+ %add.r = fadd double %sum.r, %ext.r
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ store double %add.b, ptr %out, align 8
+ %out1 = getelementptr inbounds double, ptr %out, i64 1
+ store double %add.g, ptr %out1, align 8
+ %out2 = getelementptr inbounds double, ptr %out, i64 2
+ store double %add.r, ptr %out2, align 8
+ ret void
+}
+
; Check that the increased low-VF interleaved-store cost prevents selection of
; an SVE epilogue.
@@ -85,7 +127,7 @@ exit:
; CHECK: LV: Selecting VF: vscale x 16
; CHECK: LEV: Vectorizing epilogue loop with VF = 8
define void @deinterleave4_nxv4i16_load_interleave4_nxv4i8_store(
- ptr readonly %src, ptr writeonly %out, i32 %n) #0 {
+ ptr readonly %src, ptr writeonly %out, i32 %n) #0 {
entry:
%empty = icmp eq i32 %n, 0
br i1 %empty, label %exit, label %loop
@@ -130,4 +172,45 @@ exit:
ret void
}
+; CHECK-LABEL: LV: Checking a loop in 'deinterleave3_nxv3i16_load_interleave3_nxv3i16_store'
+; CHECK: Cost of 3 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%ptr.b>
+; CHECK: Cost of 3 for VF vscale x 8: INTERLEAVE-GROUP with factor 3, ir<%out.b>
+define void @deinterleave3_nxv3i16_load_interleave3_nxv3i16_store(
+ ptr readonly %src, ptr writeonly %out, i32 %n) #0 {
+entry:
+ %empty = icmp eq i32 %n, 0
+ br i1 %empty, label %exit, label %loop
+
+loop:
+ %src.iv = phi ptr [ %src.next, %loop ], [ %src, %entry ]
+ %out.iv = phi ptr [ %out.next, %loop ], [ %out, %entry ]
+ %iv = phi i32 [ %iv.next, %loop ], [ %n, %entry ]
+
+ %ptr.b = getelementptr inbounds { i16, i16, i16 }, ptr %src.iv, i64 0, i32 0
+ %ptr.g = getelementptr inbounds { i16, i16, i16 }, ptr %src.iv, i64 0, i32 1
+ %ptr.r = getelementptr inbounds { i16, i16, i16 }, ptr %src.iv, i64 0, i32 2
+ %load.b = load i16, ptr %ptr.b, align 2
+ %load.g = load i16, ptr %ptr.g, align 2
+ %load.r = load i16, ptr %ptr.r, align 2
+
+ %shift.b = lshr i16 %load.b, 8
+ %shift.g = lshr i16 %load.g, 8
+ %shift.r = lshr i16 %load.r, 8
+ %out.b = getelementptr inbounds { i16, i16, i16 }, ptr %out.iv, i64 0, i32 0
+ %out.g = getelementptr inbounds { i16, i16, i16 }, ptr %out.iv, i64 0, i32 1
+ %out.r = getelementptr inbounds { i16, i16, i16 }, ptr %out.iv, i64 0, i32 2
+ store i16 %shift.b, ptr %out.b, align 2
+ store i16 %shift.g, ptr %out.g, align 2
+ store i16 %shift.r, ptr %out.r, align 2
+
+ %src.next = getelementptr inbounds { i16, i16, i16 }, ptr %src.iv, i64 1
+ %out.next = getelementptr inbounds { i16, i16, i16 }, ptr %out.iv, i64 1
+ %iv.next = add nsw i32 %iv, -1
+ %done = icmp eq i32 %iv.next, 0
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 26ca4b60a00de..a2681ddab7db6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -1311,9 +1311,6 @@ end:
; dst[i].y = a[i].y - b[i].y;
; dst[i].z = a[i].z << b[i].z;
; }
-;
-; TODO: Support scalable interleave groups once we can also codegen
-; @llvm.[de]interleave3
%struct.xyz = type { i32, i32, i32 }
define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr readonly %a, ptr readonly %b) {
@@ -1326,36 +1323,28 @@ define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr rea
; CHECK: vector.ph:
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP3:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP1]], i64 0
-; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP1:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP1]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 12 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC1]])
+; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 2
; CHECK-NEXT: [[TMP4:%.*]] = add nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER2]], [[WIDE_MASKED_GATHER]]
-; CHECK-NEXT: [[WIDE_GEP3:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x ptr> align 4 [[WIDE_GEP3]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[WIDE_GEP4:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 4
-; CHECK-NEXT: [[WIDE_MASKED_GATHER5:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP4]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP6:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 4
-; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP6]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP5:%.*]] = sub nsw <vscale x 4 x i32> [[WIDE_MASKED_GATHER5]], [[WIDE_MASKED_GATHER7]]
-; CHECK-NEXT: [[WIDE_GEP8:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 4
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP5]], <vscale x 4 x ptr> align 4 [[WIDE_GEP8]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[WIDE_GEP9:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP]], i64 8
-; CHECK-NEXT: [[WIDE_MASKED_GATHER10:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP9]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[WIDE_GEP11:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP1]], i64 8
-; CHECK-NEXT: [[WIDE_MASKED_GATHER12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[WIDE_GEP11]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP6:%.*]] = shl <vscale x 4 x i32> [[WIDE_MASKED_GATHER10]], [[WIDE_MASKED_GATHER12]]
-; CHECK-NEXT: [[WIDE_GEP13:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[WIDE_GEP3]], i64 8
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP6]], <vscale x 4 x ptr> align 4 [[WIDE_GEP13]], <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]])
+; CHECK-NEXT: store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP12]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP33:![0-9]+]]
; CHECK: middle.block:
More information about the llvm-commits
mailing list