[llvm] [AArch64][LV] Adjust costs for low-VF interleaved access (PR #209441)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 15 06:03:50 PDT 2026
https://github.com/jacob-crawley updated https://github.com/llvm/llvm-project/pull/209441
>From 3d453d92ce4e5bbc9dc81b2ed23daaa00a20989c Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 14 Jul 2026 11:02:29 +0000
Subject: [PATCH 1/2] [AArch64][LV] Adjust costs for low-VF interleaved access
Addressing regression introduced by #205844 in which a significantly
slower SVE tail loop is generated.
The cost model for the case where the interleave factor is larger than
the VF has been adjusted to more accurately reflect the cost of the uzp
instructions generated by the deinterleave tree, and the cost of
legalizing the type of each subvector.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 20 +++--
.../AArch64/sve-interleave-low-vf-cost.ll | 79 ++++++++++++++++---
.../AArch64/sve-interleaved-access-low-vf.ll | 2 +-
3 files changed, 84 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 9e1a0960617a5..20ffb1d9c342e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5460,13 +5460,19 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
}
// llvm.vector.deinterleaveN is lowered as a binary tree of deinterleave2
- // operations. A binary tree producing Factor leaf vectors has
- // (Factor -1) inner deinterleave2 nodes. Each deinterleave2 on a pair of
- // SVE registers emits one uzp1 + one uzp2.
- // Total shuffle cost: (Factor - 1) deinterleave2 operations, each
- // processing LT.first legal vector parts,with one uzp shuffle per part.
- auto LT = getTypeLegalizationCost(VecTy);
- return MemCost + (Factor - 1) * LT.first;
+ // operations. The tree has Log2(Factor) levels, with Factor UZP/ZIP
+ // operations at each level, giving a total shuffle cost of
+ // Factor * Log2(Factor).
+
+ // For stores, account for an additional legalization cost when
+ // repacking the legalized subvectors into the narrow interleaved
+ // vector.
+ auto LegalizationCost = getTypeLegalizationCost(SubVecTy).first;
+
+ if (Opcode == Instruction::Store)
+ LegalizationCost *= 2;
+
+ return MemCost + (Factor * LegalizationCost) + (Factor * Log2_64(Factor));
}
}
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
index 9e99bedb9ed0c..fde4e52737292 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleave-low-vf-cost.ll
@@ -5,17 +5,18 @@ target triple = "aarch64"
; Cost-model test for the path where the interleave factor is > the VF.
; The cost is modelled as a contiguous load of the wide vector plus a
-; vector.deinterleave4 lowered as a binary tree of (Factor - 1) deinterleave2 shuffles:
+; vector.deinterleave4 lowered as a binary tree of Log2(Factor) levels,
+; each with Factor operations.
;
-; cost = MemCost + (Factor - 1) * LT.first
-; * VF vscale x 2: wide type <vscale x 8 x i16>, LT.first = 1
-; => 1 (load) + 3 * 1 (shuffles) = 4
-; * VF vscale x 4: wide type <vscale x 16 x i16> LT.first = 2
-; => 2 (loads) + 3 * 2 (shuffles) = 8
+; cost = MemCost + (Factor * LegalizationCost) + (Factor * Log2(Factor))
+; * VF vscale x 2: wide type <vscale x 8 x i16>, MemCost = 1, Subvector Legalization Cost = 1
+; => 1 + (4 * 1) + (4 * 2) = 13
+; * VF vscale x 4: wide type <vscale x 16 x i16>, MemCost = 2, Subvector Legalization Cost = 1
+; => 2 + (4 * 1) + (4 * 2) = 14
; CHECK-LABEL: LV: Checking a loop in 'deinterleave4_nxv2i16_load'
-; CHECK: Cost of 4 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%ptr.b>
-; CHECK: Cost of 8 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%ptr.b>
+; CHECK: Cost of 13 for VF vscale x 2: INTERLEAVE-GROUP with factor 4, ir<%ptr.b>
+; CHECK: Cost of 14 for VF vscale x 4: INTERLEAVE-GROUP with factor 4, ir<%ptr.b>
; CHECK: LV: Selecting VF: vscale x 2
define void @deinterleave4_nxv2i16_load(ptr noalias readonly %src, ptr noalias %out, i64 %n) #0 {
entry:
@@ -69,4 +70,64 @@ exit:
ret void
}
-attributes #0 = { "target-features"="+sve" }
\ No newline at end of file
+; Check that the increased low-VF interleaved-store cost prevents selection of
+; an SVE epilogue.
+
+; For VF vscale x 4:
+; load cost = 2 + (4 * 1) + (4 * 2) = 14
+; store cost = 1 + (4 * 2) + (4 * 2) = 17
+;
+; This makes the fixed VF 8 epilogue preferable to VF vscale x 4.
+;
+; CHECK-LABEL: LV: Checking a loop in 'deinterleave4_nxv4i16_load_interleave4_nxv4i8_store'
+; CHECK: Cost of 14 for VF vscale x 4: INTERLEAVE-GROUP with factor 4
+; CHECK: Cost of 17 for VF vscale x 4: INTERLEAVE-GROUP with factor 4
+; CHECK: LV: Selecting VF: vscale x 16
+; CHECK: LEV: Vectorizing epilogue loop with VF = 8
+define void @deinterleave4_nxv4i16_load_interleave4_nxv4i8_store(
+ ptr readonly %src, ptr writeonly %out, i32 %n) #0 {
+entry:
+ %empty = icmp eq i32 %n, 0
+ br i1 %empty, label %exit, label %loop
+
+loop:
+ %src.iv = phi ptr [ %src.next, %loop ], [ %src, %entry ]
+ %out.iv = phi ptr [ %out.next, %loop ], [ %out, %entry ]
+ %iv = phi i32 [ %iv.next, %loop ], [ %n, %entry ]
+
+ %ptr.g = getelementptr inbounds i16, ptr %src.iv, i64 1
+ %ptr.r = getelementptr inbounds i16, ptr %src.iv, i64 2
+ %ptr.a = getelementptr inbounds i16, ptr %src.iv, i64 3
+ %load.b = load i16, ptr %src.iv, align 2
+ %load.g = load i16, ptr %ptr.g, align 2
+ %load.r = load i16, ptr %ptr.r, align 2
+ %load.a = load i16, ptr %ptr.a, align 2
+
+ %shift.b = lshr i16 %load.b, 8
+ %shift.g = lshr i16 %load.g, 8
+ %shift.r = lshr i16 %load.r, 8
+ %shift.a = lshr i16 %load.a, 8
+ %trunc.b = trunc nuw i16 %shift.b to i8
+ %trunc.g = trunc nuw i16 %shift.g to i8
+ %trunc.r = trunc nuw i16 %shift.r to i8
+ %trunc.a = trunc nuw i16 %shift.a to i8
+
+ %out.g = getelementptr inbounds i8, ptr %out.iv, i64 1
+ %out.r = getelementptr inbounds i8, ptr %out.iv, i64 2
+ %out.a = getelementptr inbounds i8, ptr %out.iv, i64 3
+ store i8 %trunc.b, ptr %out.iv, align 1
+ store i8 %trunc.g, ptr %out.g, align 1
+ store i8 %trunc.r, ptr %out.r, align 1
+ store i8 %trunc.a, ptr %out.a, align 1
+
+ %src.next = getelementptr inbounds i16, ptr %src.iv, i64 4
+ %out.next = getelementptr inbounds i8, ptr %out.iv, i64 4
+ %iv.next = add nsw i32 %iv, -1
+ %done = icmp eq i32 %iv.next, 0
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-access-low-vf.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-access-low-vf.ll
index 035a9f80fae4a..7b6fe0a9c6b1f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-access-low-vf.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-access-low-vf.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
; RUN: opt -passes=loop-vectorize -S < %s | FileCheck %s
target triple = "aarch64"
>From 7581f5eb1f8efd9a051d6a3877e34747c976ec7b Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 15 Jul 2026 13:00:50 +0000
Subject: [PATCH 2/2] Move store comment
---
llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 20ffb1d9c342e..7df52e1903a69 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5463,12 +5463,12 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
// operations. The tree has Log2(Factor) levels, with Factor UZP/ZIP
// operations at each level, giving a total shuffle cost of
// Factor * Log2(Factor).
+ llvm::InstructionCost LegalizationCost =
+ getTypeLegalizationCost(SubVecTy).first;
// For stores, account for an additional legalization cost when
// repacking the legalized subvectors into the narrow interleaved
// vector.
- auto LegalizationCost = getTypeLegalizationCost(SubVecTy).first;
-
if (Opcode == Instruction::Store)
LegalizationCost *= 2;
More information about the llvm-commits
mailing list