[llvm] [AArch64][LV] Cost low-VF interleaved access (PR #205844)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 25 08:04:30 PDT 2026
https://github.com/jacob-crawley created https://github.com/llvm/llvm-project/pull/205844
Adds a case to getInterleavedMemoryOpCost to cost scalable interleaved
memory accesses where the interleave factor is larger than the VF.
Previously for these cases, memory operations would be costed as
individual gathers and scatters, which may result in the LoopVectorizer
choosing a wider VF than necessary.
This changes proposes an alternative approach of using a contiguous
load/store of the interleaved vector followed by shuffles to get
the elements into place.
>From cabf836051a176053e90f0f949215455a466496f Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Mon, 8 Jun 2026 11:11:36 +0000
Subject: [PATCH 1/2] [AArch64][LV] Cost low-VF interleaved access
Adds a case to getInterleavedMemoryOpCost to cost scalable interleaved
memory accesses where the interleave factor is larger than the VF.
Previously for these cases, memory operations would be costed as
individual gathers and scatters which may result in the LoopVectorizer
choosing a wider VF than necessary.
This changes proposes an alternative approach of using a contiguous
load/store of the interleaved vector followed by shuffles to get
the elements into place.
---
.../AArch64/AArch64TargetTransformInfo.cpp | 39 +++++++++++++++++++
1 file changed, 39 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 57ae3c75c07fa..bd91f58acc925 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5244,6 +5244,45 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
if (MinElts % Factor == 0 &&
TLI->isLegalInterleavedAccessType(SubVecTy, DL, UseScalable))
return Factor * TLI->getNumInterleavedAccesses(SubVecTy, DL, UseScalable);
+
+ // Cost the alternative approach for scalable vectors where the interleave
+ // factor is larger than the VF: use a contiguous load/store of the full
+ // wide vector followed by deinterleave/interleave shuffles.
+ if (VecTy->isScalableTy() && MinElts % Factor == 0) {
+ unsigned SubVecElts = SubVecTy->getElementCount().getKnownMinValue();
+
+ if (SubVecElts < 2)
+ return InstructionCost::getInvalid();
+
+ // Cost of the contiguous memory operation on the wide vector.
+ InstructionCost MemCost;
+ if (UseMaskForCond) {
+ unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
+ : Intrinsic::masked_store;
+ MemCost = getMemIntrinsicInstrCost(
+ MemIntrinsicCostAttributes(IID, VecTy, Alignment, AddressSpace),
+ CostKind);
+ } else {
+ MemCost =
+ getMemoryOpCost(Opcode, VecTy, Alignment, AddressSpace, CostKind);
+ }
+
+ if (!MemCost.isValid())
+ return InstructionCost::getInvalid();
+
+ // llvm.vector.deinterleaveN is lowered as log2(Factor) deinterleave2
+ // operations. Each deinterleave2 on a pair of SVE registers emits one
+ // uzp1 + one uzp2.
+ auto LT = getTypeLegalizationCost(VecTy);
+ if (!LT.first.isValid())
+ return InstructionCost::getInvalid();
+
+ // Total shuffle cost: log2(Factor) levels, each processing
+ // LT.first legal vector parts, with one uzp shuffle per part.
+ InstructionCost ShuffleCost = Log2_32(Factor) * LT.first;
+
+ return MemCost + ShuffleCost;
+ }
}
return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
>From 0de5cc2ef7a1237e2ef03f4fba6a5a0442965c72 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 24 Jun 2026 15:56:46 +0000
Subject: [PATCH 2/2] Add tests
---
.../CodeGen/AArch64/sve-interleave-low-vf.ll | 234 ++++++++++++++++++
.../AArch64/sve-interleaved-access-low-vf.ll | 127 ++++++++++
2 files changed, 361 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-interleave-low-vf.ll
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-access-low-vf.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-interleave-low-vf.ll b/llvm/test/CodeGen/AArch64/sve-interleave-low-vf.ll
new file mode 100644
index 0000000000000..0b27265682d15
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-interleave-low-vf.ll
@@ -0,0 +1,234 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -O3 < %s | FileCheck %s
+
+target triple = "aarch64"
+
+; Check the backend lowering for factor-4 deinterleave when the result
+; sub-vectors are smaller than the interleave factor.
+
+define void @deinterleave4_nxv8i16_uitofp_f64(
+; CHECK-LABEL: deinterleave4_nxv8i16_uitofp_f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z0.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: uunpkhi z3.d, z2.s
+; CHECK-NEXT: uunpklo z2.d, z2.s
+; CHECK-NEXT: uunpkhi z4.d, z0.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: uzp1 z5.d, z2.d, z3.d
+; CHECK-NEXT: uzp2 z2.d, z2.d, z3.d
+; CHECK-NEXT: uzp1 z6.d, z0.d, z4.d
+; CHECK-NEXT: uzp2 z0.d, z0.d, z4.d
+; CHECK-NEXT: uzp1 z3.d, z6.d, z5.d
+; CHECK-NEXT: uzp1 z4.d, z0.d, z2.d
+; CHECK-NEXT: uzp2 z5.d, z6.d, z5.d
+; CHECK-NEXT: uzp2 z0.d, z0.d, z2.d
+; CHECK-NEXT: movi d2, #0000000000000000
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: and z3.d, z3.d, #0xffff
+; CHECK-NEXT: and z4.d, z4.d, #0xffff
+; CHECK-NEXT: and z5.d, z5.d, #0xffff
+; CHECK-NEXT: and z0.d, z0.d, #0xffff
+; CHECK-NEXT: ucvtf z3.d, p0/m, z3.d
+; CHECK-NEXT: ucvtf z4.d, p0/m, z4.d
+; CHECK-NEXT: ucvtf z5.d, p0/m, z5.d
+; CHECK-NEXT: ucvtf z0.d, p0/m, z0.d
+; CHECK-NEXT: fmul z3.d, z1.d, z3.d
+; CHECK-NEXT: fmul z4.d, z1.d, z4.d
+; CHECK-NEXT: fmul z5.d, z1.d, z5.d
+; CHECK-NEXT: fmul z0.d, z1.d, z0.d
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: fadda d2, p0, d2, z3.d
+; CHECK-NEXT: fadda d6, p0, d6, z4.d
+; CHECK-NEXT: fadda d7, p0, d7, z5.d
+; CHECK-NEXT: fadda d1, p0, d1, z0.d
+; CHECK-NEXT: mov v2.d[1], v6.d[0]
+; CHECK-NEXT: mov v7.d[1], v1.d[0]
+; CHECK-NEXT: stp q2, q7, [x0]
+; CHECK-NEXT: ret
+ <vscale x 8 x i16> %wide.vec, <vscale x 2 x double> %mul, ptr %out) #0 {
+ %d = call { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> }
+ @llvm.vector.deinterleave4.nxv8i16(<vscale x 8 x i16> %wide.vec)
+ %b = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 0
+ %g = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 1
+ %r = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 2
+ %a = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 3
+ %fb = uitofp <vscale x 2 x i16> %b to <vscale x 2 x double>
+ %fg = uitofp <vscale x 2 x i16> %g to <vscale x 2 x double>
+ %fr = uitofp <vscale x 2 x i16> %r to <vscale x 2 x double>
+ %fa = uitofp <vscale x 2 x i16> %a to <vscale x 2 x double>
+ %mb = fmul <vscale x 2 x double> %mul, %fb
+ %mg = fmul <vscale x 2 x double> %mul, %fg
+ %mr = fmul <vscale x 2 x double> %mul, %fr
+ %ma = fmul <vscale x 2 x double> %mul, %fa
+ %sb = call double @llvm.vector.reduce.fadd.nxv2f64(double 0.0, <vscale x 2 x double> %mb)
+ %sg = call double @llvm.vector.reduce.fadd.nxv2f64(double 0.0, <vscale x 2 x double> %mg)
+ %sr = call double @llvm.vector.reduce.fadd.nxv2f64(double 0.0, <vscale x 2 x double> %mr)
+ %sa = call double @llvm.vector.reduce.fadd.nxv2f64(double 0.0, <vscale x 2 x double> %ma)
+ store double %sb, ptr %out, align 8
+ %out.1 = getelementptr inbounds double, ptr %out, i64 1
+ store double %sg, ptr %out.1, align 8
+ %out.2 = getelementptr inbounds double, ptr %out, i64 2
+ store double %sr, ptr %out.2, align 8
+ %out.3 = getelementptr inbounds double, ptr %out, i64 3
+ store double %sa, ptr %out.3, align 8
+ ret void
+}
+
+
+define void @deinterleave4_nxv8i16_zext_i64(
+; CHECK-LABEL: deinterleave4_nxv8i16_zext_i64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z1.s, z0.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpkhi z2.d, z1.s
+; CHECK-NEXT: uunpklo z1.d, z1.s
+; CHECK-NEXT: uunpkhi z3.d, z0.s
+; CHECK-NEXT: uunpklo z0.d, z0.s
+; CHECK-NEXT: uzp1 z4.d, z1.d, z2.d
+; CHECK-NEXT: uzp2 z1.d, z1.d, z2.d
+; CHECK-NEXT: uzp1 z5.d, z0.d, z3.d
+; CHECK-NEXT: uzp2 z0.d, z0.d, z3.d
+; CHECK-NEXT: uzp1 z2.d, z5.d, z4.d
+; CHECK-NEXT: uzp1 z3.d, z0.d, z1.d
+; CHECK-NEXT: uzp2 z4.d, z5.d, z4.d
+; CHECK-NEXT: uzp2 z0.d, z0.d, z1.d
+; CHECK-NEXT: and z2.d, z2.d, #0xffff
+; CHECK-NEXT: and z3.d, z3.d, #0xffff
+; CHECK-NEXT: and z4.d, z4.d, #0xffff
+; CHECK-NEXT: and z0.d, z0.d, #0xffff
+; CHECK-NEXT: str z2, [x0]
+; CHECK-NEXT: str z3, [x0, #1, mul vl]
+; CHECK-NEXT: str z4, [x0, #2, mul vl]
+; CHECK-NEXT: str z0, [x0, #3, mul vl]
+; CHECK-NEXT: ret
+ <vscale x 8 x i16> %wide.vec, ptr %out) #0 {
+ %d = call { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> }
+ @llvm.vector.deinterleave4.nxv8i16(<vscale x 8 x i16> %wide.vec)
+ %b = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 0
+ %g = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 1
+ %r = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 2
+ %a = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 3
+ %eb = zext <vscale x 2 x i16> %b to <vscale x 2 x i64>
+ %eg = zext <vscale x 2 x i16> %g to <vscale x 2 x i64>
+ %er = zext <vscale x 2 x i16> %r to <vscale x 2 x i64>
+ %ea = zext <vscale x 2 x i16> %a to <vscale x 2 x i64>
+ store <vscale x 2 x i64> %eb, ptr %out, align 8
+ %out.1 = getelementptr inbounds <vscale x 2 x i64>, ptr %out, i64 1
+ store <vscale x 2 x i64> %eg, ptr %out.1, align 8
+ %out.2 = getelementptr inbounds <vscale x 2 x i64>, ptr %out, i64 2
+ store <vscale x 2 x i64> %er, ptr %out.2, align 8
+ %out.3 = getelementptr inbounds <vscale x 2 x i64>, ptr %out, i64 3
+ store <vscale x 2 x i64> %ea, ptr %out.3, align 8
+ ret void
+}
+
+; Shuffle optimization pass should be used for the in loop
+; test and replace uunpk/uzp with tbl shuffles.
+
+define void @deinterleave4_nxv8i16_zext_i64_in_loop(
+; CHECK-LABEL: deinterleave4_nxv8i16_zext_i64_in_loop:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: index z7.d, #0, #4
+; CHECK-NEXT: mov z4.d, #0xffffffffffff0000
+; CHECK-NEXT: mov z5.d, #0xffffffffffff0001
+; CHECK-NEXT: mov x9, #-65534 // =0xffffffffffff0002
+; CHECK-NEXT: mov z16.d, #0xffffffffffff0003
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z6.d, x9
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: mov x8, xzr
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: cnth x9
+; CHECK-NEXT: add z4.d, z7.d, z4.d
+; CHECK-NEXT: add z5.d, z7.d, z5.d
+; CHECK-NEXT: ptrue p0.h
+; CHECK-NEXT: add z6.d, z7.d, z6.d
+; CHECK-NEXT: add z7.d, z7.d, z16.d
+; CHECK-NEXT: .LBB2_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: ld1h { z16.h }, p0/z, [x0, x8, lsl #1]
+; CHECK-NEXT: add x8, x8, x9
+; CHECK-NEXT: cmp x8, x2
+; CHECK-NEXT: tbl z17.h, { z16.h }, z4.h
+; CHECK-NEXT: tbl z18.h, { z16.h }, z5.h
+; CHECK-NEXT: tbl z19.h, { z16.h }, z6.h
+; CHECK-NEXT: tbl z16.h, { z16.h }, z7.h
+; CHECK-NEXT: add z0.d, z0.d, z17.d
+; CHECK-NEXT: add z1.d, z1.d, z18.d
+; CHECK-NEXT: add z2.d, z2.d, z19.d
+; CHECK-NEXT: add z3.d, z3.d, z16.d
+; CHECK-NEXT: b.lo .LBB2_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: str z0, [x1]
+; CHECK-NEXT: str z1, [x1, #1, mul vl]
+; CHECK-NEXT: str z2, [x1, #2, mul vl]
+; CHECK-NEXT: str z3, [x1, #3, mul vl]
+; CHECK-NEXT: ret
+ ptr noalias %in, ptr noalias %out, i64 %n) #0 {
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %acc.b = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %add.b, %loop ]
+ %acc.g = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %add.g, %loop ]
+ %acc.r = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %add.r, %loop ]
+ %acc.a = phi <vscale x 2 x i64> [ zeroinitializer, %entry ], [ %add.a, %loop ]
+ %gep = getelementptr inbounds i16, ptr %in, i64 %iv
+ %wide.vec = load <vscale x 8 x i16>, ptr %gep, align 2
+ %d = call { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> }
+ @llvm.vector.deinterleave4.nxv8i16(<vscale x 8 x i16> %wide.vec)
+ %b = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 0
+ %g = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 1
+ %r = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 2
+ %a = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>,
+ <vscale x 2 x i16>, <vscale x 2 x i16> } %d, 3
+ %eb = zext <vscale x 2 x i16> %b to <vscale x 2 x i64>
+ %eg = zext <vscale x 2 x i16> %g to <vscale x 2 x i64>
+ %er = zext <vscale x 2 x i16> %r to <vscale x 2 x i64>
+ %ea = zext <vscale x 2 x i16> %a to <vscale x 2 x i64>
+ %add.b = add <vscale x 2 x i64> %acc.b, %eb
+ %add.g = add <vscale x 2 x i64> %acc.g, %eg
+ %add.r = add <vscale x 2 x i64> %acc.r, %er
+ %add.a = add <vscale x 2 x i64> %acc.a, %ea
+ %vscale = call i64 @llvm.vscale.i64()
+ %step = shl nuw nsw i64 %vscale, 3
+ %iv.next = add nuw nsw i64 %iv, %step
+ %ec = icmp ult i64 %iv.next, %n
+ br i1 %ec, label %loop, label %exit
+
+exit:
+ store <vscale x 2 x i64> %add.b, ptr %out, align 8
+ %out.1 = getelementptr inbounds <vscale x 2 x i64>, ptr %out, i64 1
+ store <vscale x 2 x i64> %add.g, ptr %out.1, align 8
+ %out.2 = getelementptr inbounds <vscale x 2 x i64>, ptr %out, i64 2
+ store <vscale x 2 x i64> %add.r, ptr %out.2, align 8
+ %out.3 = getelementptr inbounds <vscale x 2 x i64>, ptr %out, i64 3
+ store <vscale x 2 x i64> %add.a, ptr %out.3, align 8
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
+
+declare { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> }
+ @llvm.vector.deinterleave4.nxv8i16(<vscale x 8 x i16>)
+declare double @llvm.vector.reduce.fadd.nxv2f64(double, <vscale x 2 x double>)
+declare i64 @llvm.vscale.i64()
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-access-low-vf.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-access-low-vf.ll
new file mode 100644
index 0000000000000..09ad31dc60377
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-access-low-vf.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-vectorize -mtriple=aarch64-linux-gnu -mattr=+sve -S < %s | FileCheck %s
+
+; Check that a factor-4 interleaved i16 load can still be vectorized with a
+; scalable VF smaller than the interleave factor. Since ld4 is not legal for the
+; resulting <vscale x 2 x i16> subvectors, LV should use a contiguous
+; <vscale x 8 x i16> load followed by vector.deinterleave4.
+
+define void @uitofp_nxv8i16_to_nxv8f64_deinterleave(ptr noalias readonly %src, ptr noalias %out, i64 %n) #0 {
+; CHECK-LABEL: define void @uitofp_nxv8i16_to_nxv8f64_deinterleave(
+; CHECK-SAME: ptr noalias readonly [[SRC:%.*]], ptr noalias [[OUT:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP28:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP30:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP32:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi double [ 0.000000e+00, %[[VECTOR_PH]] ], [ [[TMP34:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP2]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = shl nuw i64 [[INDEX]], 2
+; CHECK-NEXT: [[TMP8:%.*]] = shl nuw i64 [[TMP6]], 2
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[SRC]], i64 [[TMP8]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 8 x i16>, ptr [[TMP9]], align 2
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4.nxv8i16(<vscale x 8 x i16> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } [[STRIDED_VEC]], 3
+; CHECK-NEXT: [[WIDE_VEC4:%.*]] = load <vscale x 8 x i16>, ptr [[TMP10]], align 2
+; CHECK-NEXT: [[STRIDED_VEC5:%.*]] = call { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } @llvm.vector.deinterleave4.nxv8i16(<vscale x 8 x i16> [[WIDE_VEC4]])
+; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } [[STRIDED_VEC5]], 0
+; CHECK-NEXT: [[TMP16:%.*]] = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } [[STRIDED_VEC5]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } [[STRIDED_VEC5]], 2
+; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16>, <vscale x 2 x i16> } [[STRIDED_VEC5]], 3
+; CHECK-NEXT: [[TMP19:%.*]] = uitofp <vscale x 2 x i16> [[TMP11]] to <vscale x 2 x double>
+; CHECK-NEXT: [[TMP20:%.*]] = uitofp <vscale x 2 x i16> [[TMP15]] to <vscale x 2 x double>
+; CHECK-NEXT: [[TMP21:%.*]] = uitofp <vscale x 2 x i16> [[TMP12]] to <vscale x 2 x double>
+; CHECK-NEXT: [[TMP22:%.*]] = uitofp <vscale x 2 x i16> [[TMP16]] to <vscale x 2 x double>
+; CHECK-NEXT: [[TMP23:%.*]] = uitofp <vscale x 2 x i16> [[TMP13]] to <vscale x 2 x double>
+; CHECK-NEXT: [[TMP24:%.*]] = uitofp <vscale x 2 x i16> [[TMP17]] to <vscale x 2 x double>
+; CHECK-NEXT: [[TMP25:%.*]] = uitofp <vscale x 2 x i16> [[TMP14]] to <vscale x 2 x double>
+; CHECK-NEXT: [[TMP26:%.*]] = uitofp <vscale x 2 x i16> [[TMP18]] to <vscale x 2 x double>
+; CHECK-NEXT: [[TMP27:%.*]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], <vscale x 2 x double> [[TMP19]])
+; CHECK-NEXT: [[TMP28]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP27]], <vscale x 2 x double> [[TMP20]])
+; CHECK-NEXT: [[TMP29:%.*]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI1]], <vscale x 2 x double> [[TMP21]])
+; CHECK-NEXT: [[TMP30]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP29]], <vscale x 2 x double> [[TMP22]])
+; CHECK-NEXT: [[TMP31:%.*]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI2]], <vscale x 2 x double> [[TMP23]])
+; CHECK-NEXT: [[TMP32]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP31]], <vscale x 2 x double> [[TMP24]])
+; CHECK-NEXT: [[TMP33:%.*]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI3]], <vscale x 2 x double> [[TMP25]])
+; CHECK-NEXT: [[TMP34]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP33]], <vscale x 2 x double> [[TMP26]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP35]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %sum.b = phi double [ 0.0, %entry ], [ %add.b, %loop ]
+ %sum.g = phi double [ 0.0, %entry ], [ %add.g, %loop ]
+ %sum.r = phi double [ 0.0, %entry ], [ %add.r, %loop ]
+ %sum.a = phi double [ 0.0, %entry ], [ %add.a, %loop ]
+
+ %base = shl nuw i64 %iv, 2
+ %ptr.b = getelementptr inbounds i16, ptr %src, i64 %base
+ %load.b = load i16, ptr %ptr.b, align 2
+
+ %off.g = add nuw i64 %base, 1
+ %ptr.g = getelementptr inbounds i16, ptr %src, i64 %off.g
+ %load.g = load i16, ptr %ptr.g, align 2
+
+ %off.r = add nuw i64 %base, 2
+ %ptr.r = getelementptr inbounds i16, ptr %src, i64 %off.r
+ %load.r = load i16, ptr %ptr.r, align 2
+
+ %off.a = add nuw i64 %base, 3
+ %ptr.a = getelementptr inbounds i16, ptr %src, i64 %off.a
+ %load.a = load i16, ptr %ptr.a, align 2
+
+ %ext.b = uitofp i16 %load.b to double
+ %ext.g = uitofp i16 %load.g to double
+ %ext.r = uitofp i16 %load.r to double
+ %ext.a = uitofp i16 %load.a to double
+
+ %add.b = fadd double %sum.b, %ext.b
+ %add.g = fadd double %sum.g, %ext.g
+ %add.r = fadd double %sum.r, %ext.r
+ %add.a = fadd double %sum.a, %ext.a
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ %res.b = phi double [ %add.b, %loop ]
+ %res.g = phi double [ %add.g, %loop ]
+ %res.r = phi double [ %add.r, %loop ]
+ %res.a = phi double [ %add.a, %loop ]
+ store double %res.b, ptr %out, align 8
+ %out1 = getelementptr inbounds double, ptr %out, i64 1
+ store double %res.g, ptr %out1, align 8
+ %out2 = getelementptr inbounds double, ptr %out, i64 2
+ store double %res.r, ptr %out2, align 8
+ %out3 = getelementptr inbounds double, ptr %out, i64 3
+ store double %res.a, ptr %out3, align 8
+ ret void
+}
+
+attributes #0 = { "target-features"="+sve" }
More information about the llvm-commits
mailing list