[llvm-branch-commits] [llvm] [SLP] Price gathered lanes of vectorized entries as a shuffle (PR #240136)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Nov 8 05:14:10 PST 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: Dmitry Sidorov (MrSidims)
<details>
<summary>Changes</summary>
A gathered lane already vectorized in another entry is moved out of
that vector with a shuffle, which a zero insert price misses. Add the
shuffle cost of these lanes when their insert price is zero.
---
Patch is 43.75 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/240136.diff
13 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+20)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/gathered-lane-of-vectorized-loads.ll (+8-5)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll (+1-2)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-operand-deps-direct-use.ll (+35-32)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-subtree-schedule-operands.ll (+14-15)
- (modified) llvm/test/Transforms/SLPVectorizer/AMDGPU/gathered-load-strided-picks.ll (+1-1)
- (added) llvm/test/Transforms/SLPVectorizer/RISCV/gathered-lane-of-vectorized-entry-zve32.ll (+34)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll (+7-6)
- (modified) llvm/test/Transforms/SLPVectorizer/absorbed-lane-shuffled-operand.ll (+1-2)
- (modified) llvm/test/Transforms/SLPVectorizer/extract-many-users-buildvector.ll (+21-22)
- (modified) llvm/test/Transforms/SLPVectorizer/gathered-consecutive-loads-different-types.ll (+4-4)
- (modified) llvm/test/Transforms/SLPVectorizer/gathered-swapped-vectorized-lanes.ll (+14-12)
- (modified) llvm/test/Transforms/SLPVectorizer/resized-alt-shuffle-after-minbw.ll (+6-6)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index fbef70d8aff522..9c0ceeb13f93ef 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -21932,6 +21932,26 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
getScalarizationOverhead(*TTI, Opts.slp_revec, VL.front()->getType(),
MaskVecTy, DemandedElts, /*Insert=*/true,
/*Extract=*/false, CostKind);
+ // A lane taken from another vectorized entry needs a shuffle, which a zero
+ // insert price misses.
+ APInt VectorizedElts = APInt::getZero(SubMask.size());
+ SmallVector<int> VectorizedMask(SubMask.size(), PoisonMaskElem);
+ for (auto [I, Idx] : enumerate(SubMask)) {
+ if (Idx == PoisonMaskElem || !isVectorized(VL[I]))
+ continue;
+ VectorizedElts.setBit(I);
+ VectorizedMask[I] = Idx;
+ }
+ if (!VectorizedElts.isZero() &&
+ getScalarizationOverhead(*TTI, Opts.slp_revec, VL.front()->getType(),
+ MaskVecTy, VectorizedElts, /*Insert=*/true,
+ /*Extract=*/false,
+ CostKind) == TTI::TCC_Free) {
+ InstructionCost VectorizedShuffleCost =
+ GetShuffleCost(VectorizedMask, Entries, VecTy);
+ if (VectorizedShuffleCost.isValid())
+ BuildVectorCost += VectorizedShuffleCost;
+ }
const TreeEntry *BestEntry = nullptr;
auto MaskSlice = MutableArrayRef(Mask).slice(MaskBase, VL.size());
if (FirstShuffleCost < ShuffleCost) {
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gathered-lane-of-vectorized-loads.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gathered-lane-of-vectorized-loads.ll
index f365aa329cb406..e80afa9cd12cc1 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gathered-lane-of-vectorized-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gathered-lane-of-vectorized-loads.ll
@@ -6,11 +6,14 @@
define void @load_lane_and_arg(ptr %a, double %x, ptr %out) {
; CHECK-LABEL: define void @load_lane_and_arg(
; CHECK-SAME: ptr [[A:%.*]], double [[X:%.*]], ptr [[OUT:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[A]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[X]], i64 1
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> [[TMP1]], <2 x i32> <i32 3, i32 1>
-; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: store <2 x double> [[TMP4]], ptr [[OUT]], align 8
+; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[A]], i64 1
+; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[A]], align 8
+; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
+; CHECK-NEXT: [[M0:%.*]] = fadd double [[A0]], [[A1]]
+; CHECK-NEXT: [[M1:%.*]] = fadd double [[A1]], [[X]]
+; CHECK-NEXT: [[O1:%.*]] = getelementptr double, ptr [[OUT]], i64 1
+; CHECK-NEXT: store double [[M0]], ptr [[OUT]], align 8
+; CHECK-NEXT: store double [[M1]], ptr [[O1]], align 8
; CHECK-NEXT: ret void
;
%p1 = getelementptr double, ptr %a, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
index e1a4cee87e5089..983c568fb01d25 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
@@ -50,8 +50,7 @@ define double @loop_acc_fadd(ptr %p, i64 %n) {
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[C8]], i64 2
; CHECK-NEXT: [[TMP18:%.*]] = fmul reassoc nnan ninf arcp contract afn <4 x double> [[TMP14]], [[TMP17]]
; CHECK-NEXT: [[TMP19:%.*]] = fadd reassoc nnan ninf nsz arcp afn <4 x double> [[TMP12]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
; CHECK-NEXT: [[TMP22:%.*]] = fmul <4 x double> [[TMP19]], [[TMP21]]
; CHECK-NEXT: [[SLPRDX_ACC1]] = fadd reassoc nnan ninf nsz arcp afn <4 x double> [[SLPRDX_ACC]], [[TMP22]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-operand-deps-direct-use.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-operand-deps-direct-use.ll
index e28b3eed83900e..ef1dd3ee4ca0a3 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-operand-deps-direct-use.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-operand-deps-direct-use.ll
@@ -9,50 +9,53 @@ define double @test(ptr %0, double %1) {
; CHECK-NEXT: [[TMP3:%.*]] = fsub double 0.000000e+00, [[TMP2]]
; CHECK-NEXT: [[TMP4:%.*]] = fmul double [[TMP3]], 0.000000e+00
; CHECK-NEXT: store double [[TMP4]], ptr null, align 8
-; CHECK-NEXT: [[TMP5:%.*]] = fmul double 0.000000e+00, 0.000000e+00
-; CHECK-NEXT: [[TMP6:%.*]] = fmul double [[TMP3]], 0.000000e+00
-; CHECK-NEXT: [[TMP7:%.*]] = fmul double [[TMP6]], [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[TMP5]], zeroinitializer
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x double> poison, double [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> [[TMP17]], double [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP20:%.*]] = fmul <2 x double> [[TMP8]], zeroinitializer
+; CHECK-NEXT: [[TMP21:%.*]] = fadd <2 x double> [[TMP8]], zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x double> [[TMP20]], <2 x double> [[TMP21]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <2 x double> [[TMP11]], <2 x double> <double poison, double 0.000000e+00>, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP13:%.*]] = fmul <2 x double> [[TMP26]], [[TMP5]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP13]], i64 0
; CHECK-NEXT: store double [[TMP7]], ptr null, align 8
-; CHECK-NEXT: [[TMP8:%.*]] = fadd double [[TMP1]], 0.000000e+00
-; CHECK-NEXT: [[TMP9:%.*]] = fmul double [[TMP8]], 0.000000e+00
+; CHECK-NEXT: [[TMP15:%.*]] = fmul <2 x double> [[TMP11]], zeroinitializer
+; CHECK-NEXT: [[TMP16:%.*]] = fmul <2 x double> [[TMP6]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP15]], i64 1
; CHECK-NEXT: store double [[TMP9]], ptr null, align 8
-; CHECK-NEXT: [[TMP10:%.*]] = fmul double [[TMP6]], 0.000000e+00
-; CHECK-NEXT: [[TMP11:%.*]] = fadd double 0.000000e+00, 0.000000e+00
-; CHECK-NEXT: [[TMP12:%.*]] = fmul double [[TMP11]], [[TMP1]]
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> <double 0.000000e+00, double poison>, <2 x i32> <i32 2, i32 0>
+; CHECK-NEXT: [[TMP19:%.*]] = fmul <2 x double> zeroinitializer, [[TMP18]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP19]], i64 1
; CHECK-NEXT: store double [[TMP12]], ptr null, align 8
-; CHECK-NEXT: [[TMP13:%.*]] = fmul double [[TMP11]], 0.000000e+00
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: [[TMP14:%.*]] = fmul double 0.000000e+00, [[TMP4]]
-; CHECK-NEXT: [[TMP15:%.*]] = fmul double [[TMP7]], 0.000000e+00
-; CHECK-NEXT: [[TMP16:%.*]] = fadd double [[TMP15]], [[TMP4]]
-; CHECK-NEXT: [[TMP17:%.*]] = fmul double [[TMP10]], 0.000000e+00
-; CHECK-NEXT: [[TMP18:%.*]] = fadd double [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = fmul double [[TMP12]], 0.000000e+00
-; CHECK-NEXT: [[TMP20:%.*]] = fadd double [[TMP19]], 0.000000e+00
-; CHECK-NEXT: [[TMP21:%.*]] = fmul double [[TMP13]], 0.000000e+00
-; CHECK-NEXT: [[TMP22:%.*]] = fadd double [[TMP20]], [[TMP21]]
-; CHECK-NEXT: [[TMP23:%.*]] = fadd double [[TMP22]], 0.000000e+00
-; CHECK-NEXT: [[TMP24:%.*]] = fmul double [[TMP23]], 0.000000e+00
-; CHECK-NEXT: [[TMP25:%.*]] = fadd double [[TMP18]], 0.000000e+00
-; CHECK-NEXT: [[TMP26:%.*]] = fmul double [[TMP25]], 0.000000e+00
-; CHECK-NEXT: [[TMP27:%.*]] = fsub double [[TMP24]], [[TMP26]]
+; CHECK-NEXT: [[TMP22:%.*]] = fmul <2 x double> [[TMP13]], zeroinitializer
+; CHECK-NEXT: [[TMP27:%.*]] = fmul <2 x double> [[TMP15]], zeroinitializer
+; CHECK-NEXT: [[TMP24:%.*]] = fmul <2 x double> [[TMP19]], zeroinitializer
+; CHECK-NEXT: [[TMP25:%.*]] = fmul <2 x double> [[TMP16]], zeroinitializer
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[TMP0]], i64 24
-; CHECK-NEXT: store double [[TMP27]], ptr [[TMP28]], align 8
-; CHECK-NEXT: [[TMP29:%.*]] = fmul double [[TMP5]], 0.000000e+00
-; CHECK-NEXT: [[TMP30:%.*]] = fadd double [[TMP29]], 0.000000e+00
-; CHECK-NEXT: [[TMP31:%.*]] = fmul double [[TMP9]], 0.000000e+00
-; CHECK-NEXT: [[TMP32:%.*]] = fadd double [[TMP30]], [[TMP31]]
-; CHECK-NEXT: [[TMP33:%.*]] = fadd double [[TMP32]], 0.000000e+00
-; CHECK-NEXT: [[TMP34:%.*]] = fsub double [[TMP25]], [[TMP33]]
; CHECK-NEXT: [[TMP35:%.*]] = getelementptr i8, ptr [[TMP0]], i64 32
-; CHECK-NEXT: store double [[TMP34]], ptr [[TMP35]], align 8
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <2 x double> <double poison, double 0.000000e+00>, double [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP29:%.*]] = fadd <2 x double> [[TMP22]], [[TMP34]]
+; CHECK-NEXT: [[TMP30:%.*]] = fadd <2 x double> [[TMP24]], zeroinitializer
+; CHECK-NEXT: [[TMP31:%.*]] = fadd <2 x double> [[TMP29]], [[TMP27]]
+; CHECK-NEXT: [[TMP32:%.*]] = fadd <2 x double> [[TMP30]], [[TMP25]]
+; CHECK-NEXT: [[TMP38:%.*]] = fadd <2 x double> [[TMP32]], zeroinitializer
+; CHECK-NEXT: [[TMP33:%.*]] = extractelement <2 x double> [[TMP38]], i64 1
; CHECK-NEXT: [[TMP36:%.*]] = fmul double [[TMP33]], 0.000000e+00
+; CHECK-NEXT: [[TMP47:%.*]] = fadd <2 x double> [[TMP31]], zeroinitializer
+; CHECK-NEXT: [[TMP48:%.*]] = extractelement <2 x double> [[TMP47]], i64 0
+; CHECK-NEXT: [[TMP23:%.*]] = fmul double [[TMP48]], 0.000000e+00
; CHECK-NEXT: [[TMP37:%.*]] = fsub double [[TMP36]], [[TMP23]]
-; CHECK-NEXT: [[TMP38:%.*]] = getelementptr i8, ptr [[TMP0]], i64 40
-; CHECK-NEXT: store double [[TMP37]], ptr [[TMP38]], align 8
+; CHECK-NEXT: store double [[TMP37]], ptr [[TMP28]], align 8
+; CHECK-NEXT: [[TMP49:%.*]] = fmul <2 x double> [[TMP47]], <double 1.000000e+00, double 0.000000e+00>
+; CHECK-NEXT: [[TMP50:%.*]] = fsub <2 x double> [[TMP49]], [[TMP38]]
+; CHECK-NEXT: store <2 x double> [[TMP50]], ptr [[TMP35]], align 8
; CHECK-NEXT: [[TMP39:%.*]] = fmul double [[TMP7]], 0.000000e+00
; CHECK-NEXT: [[TMP40:%.*]] = fadd double [[TMP39]], [[TMP1]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP15]], i64 0
; CHECK-NEXT: [[TMP41:%.*]] = fadd double [[TMP40]], [[TMP10]]
; CHECK-NEXT: [[TMP42:%.*]] = fadd double [[TMP41]], 0.000000e+00
; CHECK-NEXT: [[TMP43:%.*]] = fadd double [[TMP14]], 0.000000e+00
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-subtree-schedule-operands.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-subtree-schedule-operands.ll
index 239672cfeda947..b529394ea0917a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-subtree-schedule-operands.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-subtree-schedule-operands.ll
@@ -19,31 +19,30 @@ define void @test(ptr %srcGrid, ptr %dstGrid, i64 %iv) {
; CHECK-NEXT: [[ARRAYIDX148:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 64
; CHECK-NEXT: [[ARRAYIDX184:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 136
; CHECK-NEXT: [[ARRAYIDX188:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 144
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[ARRAYIDX184]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = load double, ptr [[ARRAYIDX188]], align 8
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> <double poison, double 0.000000e+00>, <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> <double poison, double 0.000000e+00>, <2 x double> [[TMP1]], <2 x i32> <i32 3, i32 1>
-; CHECK-NEXT: [[TMP5:%.*]] = fadd fast <2 x double> [[TMP3]], [[TMP9]]
-; CHECK-NEXT: [[TMP6:%.*]] = fsub fast <2 x double> zeroinitializer, [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = fadd fast double 0.000000e+00, [[TMP4]]
-; CHECK-NEXT: [[MOMENTUM_Z:%.*]] = fsub fast double 0.000000e+00, [[TMP7]]
; CHECK-NEXT: [[ARRAYIDX675:%.*]] = getelementptr inbounds nuw [8 x i8], ptr [[DSTGRID]], i64 [[IV]]
; CHECK-NEXT: [[ARRAYIDX787:%.*]] = getelementptr inbounds nuw i8, ptr [[ARRAYIDX675]], i64 32216
; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[ARRAYIDX148]], align 8
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[ARRAYIDX144]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = load double, ptr [[ARRAYIDX184]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = load double, ptr [[ARRAYIDX188]], align 8
; CHECK-NEXT: [[RHO:%.*]] = fadd fast double 1.000000e+00, [[TMP8]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x double> <double poison, double 1.000000e+00>, double [[RHO]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = fdiv fast <2 x double> [[TMP6]], [[TMP13]]
+; CHECK-NEXT: [[TMP5:%.*]] = fadd fast double [[TMP3]], [[TMP4]]
+; CHECK-NEXT: [[MOMENTUM_Z:%.*]] = fsub fast double 0.000000e+00, [[TMP5]]
+; CHECK-NEXT: [[TMP6:%.*]] = fadd fast double 0.000000e+00, 0.000000e+00
+; CHECK-NEXT: [[SUB266:%.*]] = fsub fast double 0.000000e+00, [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = fadd fast double 0.000000e+00, [[TMP4]]
+; CHECK-NEXT: [[MOMENTUM_Z1:%.*]] = fsub fast double 0.000000e+00, [[TMP7]]
; CHECK-NEXT: [[DIV307:%.*]] = fdiv fast double [[MOMENTUM_Z]], [[RHO]]
-; CHECK-NEXT: [[UZ_0:%.*]] = select nsz i1 true, double [[DIV307]], double 0.000000e+00
-; CHECK-NEXT: [[TMP12:%.*]] = select <2 x i1> zeroinitializer, <2 x double> [[TMP11]], <2 x double> zeroinitializer
+; CHECK-NEXT: [[DIV306:%.*]] = fdiv fast double [[SUB266]], 1.000000e+00
+; CHECK-NEXT: [[DIV308:%.*]] = fdiv fast double [[MOMENTUM_Z1]], [[RHO]]
+; CHECK-NEXT: [[UZ_0:%.*]] = select nsz i1 true, double [[DIV308]], double 0.000000e+00
+; CHECK-NEXT: [[TMP32:%.*]] = select nsz i1 false, double [[DIV306]], double 0.000000e+00
+; CHECK-NEXT: [[TMP31:%.*]] = select nsz i1 false, double [[DIV307]], double 0.000000e+00
; CHECK-NEXT: [[MUL317:%.*]] = fmul fast double [[UZ_0]], [[UZ_0]]
; CHECK-NEXT: [[ADD318:%.*]] = fadd fast double 0.000000e+00, [[MUL317]]
; CHECK-NEXT: [[MUL319:%.*]] = fmul fast double [[ADD318]], 1.000000e+00
; CHECK-NEXT: [[MUL348:%.*]] = fmul fast double [[RHO]], 1.000000e+00
-; CHECK-NEXT: [[UX_0:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP12]])
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <2 x double> [[TMP12]], i64 0
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <2 x double> [[TMP12]], i64 1
+; CHECK-NEXT: [[UX_0:%.*]] = fadd fast double [[TMP31]], [[TMP32]]
; CHECK-NEXT: [[ADD359:%.*]] = fsub fast double [[TMP32]], [[TMP31]]
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x double> poison, double [[UX_0]], i64 0
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x double> [[TMP10]], double [[ADD359]], i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/gathered-load-strided-picks.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/gathered-load-strided-picks.ll
index a42ea4f2e1e9fa..7681511134e032 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/gathered-load-strided-picks.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/gathered-load-strided-picks.ll
@@ -10,7 +10,7 @@
; YAML-NEXT: Function: two_accumulators_f32
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'SLP vectorized with cost '
-; YAML-NEXT: - Cost: '-16'
+; YAML-NEXT: - Cost: '-8'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '17'
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/gathered-lane-of-vectorized-entry-zve32.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/gathered-lane-of-vectorized-entry-zve32.ll
new file mode 100644
index 00000000000000..7b2bab2ff2f001
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/gathered-lane-of-vectorized-entry-zve32.ll
@@ -0,0 +1,34 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=riscv64-unknown-linux-gnu -mattr=+zve32f,+zvl128b < %s | FileCheck %s
+
+; With ELEN 32 the i64 shuffle has no valid cost, so the insert price stays.
+
+define void @mul_lane_and_arg(ptr %p, ptr %q, i64 %c) {
+; CHECK-LABEL: define void @mul_lane_and_arg(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i64 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = xor <2 x i64> [[TMP1]], splat (i64 1)
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> poison, i64 [[C]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP2]], <2 x i32> <i32 3, i32 1>
+; CHECK-NEXT: [[TMP5:%.*]] = mul <2 x i64> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: store <2 x i64> [[TMP5]], ptr [[Q]], align 8
+; CHECK-NEXT: ret void
+;
+ %p1 = getelementptr inbounds i64, ptr %p, i64 1
+ %a0 = load i64, ptr %p, align 8
+ %a1 = load i64, ptr %p1, align 8
+ %x0 = xor i64 %a0, 1
+ %x1 = xor i64 %a1, 1
+ %z0 = zext i64 %x0 to i128
+ %z1 = zext i64 %x1 to i128
+ %w0 = zext i64 %x1 to i128
+ %w1 = zext i64 %c to i128
+ %m0 = mul nuw i128 %z0, %w0
+ %m1 = mul nuw i128 %z1, %w1
+ %t0 = trunc i128 %m0 to i64
+ %t1 = trunc i128 %m1 to i64
+ %q1 = getelementptr inbounds i64, ptr %q, i64 1
+ store i64 %t0, ptr %q, align 8
+ store i64 %t1, ptr %q1, align 8
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll b/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
index 396fd161aa8885..1204ae0f31254d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gathered-shuffle-resized.ll
@@ -15,12 +15,13 @@ define ptr @test(ptr %0, ptr %args_gep) {
; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[ARG26]], i64 17
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[ARG1]], i64 8
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[ARG1]], i64 12
-; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !noalias [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP9:%.*]] = load <2 x i32>, ptr [[TMP6]], align 8, !noalias [[META0]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <16 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <16 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i32> [[TMP10]], <16 x i32> [[TMP11]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 16, i32 8, i32 9, i32 10, i32 11, i32 12, i32 22, i32 14, i32 15>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i32> [[TMP12]], <16 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 13, i32 13, i32 13>
+; CHECK-NEXT: [[TMP9:%.*]] = load <2 x i32>, ptr [[TMP6]], align 8, !noalias [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4, !noalias [[...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/240136
More information about the llvm-branch-commits
mailing list