[llvm] dd19d52 - [SLP]Recalculate copyable-element deps after tree reordering
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 14:48:10 PDT 2026
Author: Alexey Bataev
Date: 2026-08-14T17:48:06-04:00
New Revision: dd19d5210352f004758d0a877b396ea9f04cfd70
URL: https://github.com/llvm/llvm-project/commit/dd19d5210352f004758d0a877b396ea9f04cfd70
DIFF: https://github.com/llvm/llvm-project/commit/dd19d5210352f004758d0a877b396ea9f04cfd70.diff
LOG: [SLP]Recalculate copyable-element deps after tree reordering
Reordering permutes the operand columns of the entries and may move an
operand between copyable-covered and plain edges, making the computed
dependency counts stale and tripping the unscheduled-deps assertion.
Reviewers:
Pull Request: https://github.com/llvm/llvm-project/pull/216410
Added:
llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 53816d49de722..66a35f9bfaa53 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4748,6 +4748,18 @@ class slpvectorizer::BoUpSLP {
return Res;
}
+ /// Reordering permutes the operand columns of the tree entries and may
+ /// move an operand between the edges covered and not covered by copyable
+ /// scheduling data, making the computed dependency counts stale. Mark the
+ /// schedule data of the copyable-element instructions for recalculation
+ /// at the next bundle scheduling.
+ void markCopyableDepsForRecalc() {
+ for (const auto &P : ScheduleCopyableDataMapByInst)
+ if (ScheduleData *SD =
+ getScheduleData(const_cast<Instruction *>(P.first)))
+ RecalcCopyableOperandDeps.insert(SD);
+ }
+
ScheduleCopyableData &addScheduleCopyableData(const EdgeInfo &EI,
Instruction *I,
int SchedulingRegionID,
@@ -8207,6 +8219,8 @@ void BoUpSLP::reorderTopToBottom() {
Mask, MaskOrder);
}
}
+ for (const auto &It : BlocksSchedules)
+ It.second->markCopyableDepsForRecalc();
}
void BoUpSLP::buildReorderableOperands(
@@ -8708,6 +8722,8 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
if (IgnoreReorder && !VectorizableTree.front()->ReorderIndices.empty() &&
VectorizableTree.front()->ReuseShuffleIndices.empty())
VectorizableTree.front()->ReorderIndices.clear();
+ for (const auto &It : BlocksSchedules)
+ It.second->markCopyableDepsForRecalc();
}
Instruction *BoUpSLP::getRootEntryInstruction(const TreeEntry &Entry) const {
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll
new file mode 100644
index 0000000000000..f7b2664e57b80
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll
@@ -0,0 +1,188 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(ptr %0, ptr %1, double %2, ptr %3) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[TMP1:%.*]], double [[TMP2:%.*]], ptr [[TMP3:%.*]]) {
+; CHECK-NEXT: br i1 true, label %[[VECTOR_PH:.*]], label %[[DOTPREHEADER511:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP5:%.*]] = load double, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = fmul double [[TMP5]], 0.000000e+00
+; CHECK-NEXT: [[TMP7:%.*]] = fadd double [[TMP6]], 0.000000e+00
+; CHECK-NEXT: [[TMP8:%.*]] = fmul double [[TMP6]], 0.000000e+00
+; CHECK-NEXT: [[TMP9:%.*]] = fmul double [[TMP8]], 0.000000e+00
+; CHECK-NEXT: [[TMP10:%.*]] = fadd double [[TMP2]], 0.000000e+00
+; CHECK-NEXT: [[TMP11:%.*]] = fadd double [[TMP10]], 1.000000e+00
+; CHECK-NEXT: [[TMP12:%.*]] = fadd double [[TMP9]], [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = fmul double [[TMP2]], 0.000000e+00
+; CHECK-NEXT: [[TMP14:%.*]] = fadd contract double [[TMP13]], 0.000000e+00
+; CHECK-NEXT: [[TMP15:%.*]] = fmul double [[TMP6]], [[TMP2]]
+; CHECK-NEXT: [[TMP16:%.*]] = fadd contract double [[TMP15]], 0.000000e+00
+; CHECK-NEXT: [[TMP17:%.*]] = load double, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = fmul double [[TMP17]], 0.000000e+00
+; CHECK-NEXT: [[TMP19:%.*]] = fmul double [[TMP18]], [[TMP2]]
+; CHECK-NEXT: [[TMP20:%.*]] = fmul double [[TMP2]], [[TMP19]]
+; CHECK-NEXT: [[TMP21:%.*]] = fadd double [[TMP20]], 0.000000e+00
+; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP23:%.*]] = fmul double [[TMP22]], 0.000000e+00
+; CHECK-NEXT: [[TMP24:%.*]] = fmul double [[TMP2]], [[TMP23]]
+; CHECK-NEXT: [[TMP25:%.*]] = fadd double [[TMP24]], 0.000000e+00
+; CHECK-NEXT: [[TMP26:%.*]] = fmul double [[TMP5]], 0.000000e+00
+; CHECK-NEXT: [[TMP27:%.*]] = fmul double [[TMP26]], 0.000000e+00
+; CHECK-NEXT: [[TMP28:%.*]] = fadd double [[TMP27]], 0.000000e+00
+; CHECK-NEXT: [[TMP29:%.*]] = fmul double [[TMP2]], [[TMP17]]
+; CHECK-NEXT: [[TMP30:%.*]] = fadd double [[TMP29]], 0.000000e+00
+; CHECK-NEXT: br label %[[DOTPREHEADER511]]
+; CHECK: [[_PREHEADER511:.*:]]
+; CHECK-NEXT: [[DOTSROA_9_0:%.*]] = phi double [ [[TMP30]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4:%.*]] ]
+; CHECK-NEXT: [[DOTSROA_7_0:%.*]] = phi double [ [[TMP28]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
+; CHECK-NEXT: [[DOTSROA_6887_0:%.*]] = phi double [ [[TMP25]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
+; CHECK-NEXT: [[DOTSROA_5_0:%.*]] = phi double [ [[TMP21]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
+; CHECK-NEXT: [[DOTSROA_4886_0:%.*]] = phi double [ [[TMP16]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
+; CHECK-NEXT: [[DOTSROA_3_0:%.*]] = phi double [ [[TMP14]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
+; CHECK-NEXT: [[DOTSROA_2885_0:%.*]] = phi double [ [[TMP12]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
+; CHECK-NEXT: [[DOTSROA_0884_0:%.*]] = phi double [ [[TMP7]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ]
+; CHECK-NEXT: br i1 true, label %[[BB31:.*]], [[DOTPREHEADER498:label %.*]]
+; CHECK: [[BB31]]:
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8
+; CHECK-NEXT: [[TMP33:%.*]] = load double, ptr [[TMP3]], align 8
+; CHECK-NEXT: [[TMP34:%.*]] = fmul double [[DOTSROA_0884_0]], [[TMP33]]
+; CHECK-NEXT: [[TMP35:%.*]] = load double, ptr [[TMP32]], align 8
+; CHECK-NEXT: [[TMP36:%.*]] = fmul double [[DOTSROA_2885_0]], [[TMP35]]
+; CHECK-NEXT: [[TMP37:%.*]] = fadd double [[TMP34]], [[TMP36]]
+; CHECK-NEXT: [[TMP38:%.*]] = fmul double [[DOTSROA_4886_0]], 0.000000e+00
+; CHECK-NEXT: [[TMP39:%.*]] = fadd double [[TMP37]], [[TMP38]]
+; CHECK-NEXT: [[TMP40:%.*]] = fadd double [[TMP39]], 0.000000e+00
+; CHECK-NEXT: [[TMP41:%.*]] = fadd double [[TMP40]], 1.000000e+00
+; CHECK-NEXT: [[TMP42:%.*]] = fmul double [[DOTSROA_2885_0]], [[TMP33]]
+; CHECK-NEXT: [[TMP43:%.*]] = fmul double [[DOTSROA_3_0]], [[TMP35]]
+; CHECK-NEXT: [[TMP44:%.*]] = fadd double [[TMP42]], [[TMP43]]
+; CHECK-NEXT: [[TMP45:%.*]] = fmul double [[DOTSROA_5_0]], 0.000000e+00
+; CHECK-NEXT: [[TMP46:%.*]] = fadd double [[TMP44]], [[TMP45]]
+; CHECK-NEXT: [[TMP47:%.*]] = fadd double [[TMP46]], [[TMP2]]
+; CHECK-NEXT: [[TMP48:%.*]] = fadd double [[TMP47]], 0.000000e+00
+; CHECK-NEXT: [[TMP49:%.*]] = fmul double [[DOTSROA_4886_0]], [[TMP33]]
+; CHECK-NEXT: [[TMP50:%.*]] = fmul double [[DOTSROA_5_0]], [[TMP35]]
+; CHECK-NEXT: [[TMP51:%.*]] = fadd double [[TMP49]], [[TMP50]]
+; CHECK-NEXT: [[TMP52:%.*]] = load double, ptr [[TMP0]], align 8
+; CHECK-NEXT: [[TMP53:%.*]] = fmul double [[DOTSROA_6887_0]], [[TMP52]]
+; CHECK-NEXT: [[TMP54:%.*]] = fadd double [[TMP51]], [[TMP53]]
+; CHECK-NEXT: [[TMP55:%.*]] = fmul double [[DOTSROA_9_0]], [[TMP2]]
+; CHECK-NEXT: [[TMP56:%.*]] = fadd double [[TMP54]], [[TMP55]]
+; CHECK-NEXT: [[TMP57:%.*]] = fadd double [[TMP56]], 0.000000e+00
+; CHECK-NEXT: [[TMP58:%.*]] = fmul double [[DOTSROA_7_0]], [[TMP33]]
+; CHECK-NEXT: [[TMP59:%.*]] = fadd double [[TMP58]], [[TMP2]]
+; CHECK-NEXT: [[TMP60:%.*]] = fmul double [[DOTSROA_9_0]], 0.000000e+00
+; CHECK-NEXT: [[TMP61:%.*]] = fadd double [[TMP59]], [[TMP60]]
+; CHECK-NEXT: [[TMP62:%.*]] = fadd double [[TMP61]], 0.000000e+00
+; CHECK-NEXT: [[TMP63:%.*]] = fadd double [[TMP62]], 1.000000e+00
+; CHECK-NEXT: br [[DOTPREHEADER498]]
+; CHECK: [[_PREHEADER498:.*:]]
+; CHECK-NEXT: [[DOTSROA_0913_0:%.*]] = phi double [ [[TMP41]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
+; CHECK-NEXT: [[DOTSROA_6915_0:%.*]] = phi double [ [[TMP48]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
+; CHECK-NEXT: [[DOTSROA_10918_0:%.*]] = phi double [ [[TMP57]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
+; CHECK-NEXT: [[DOTSROA_14921_0:%.*]] = phi double [ [[TMP63]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
+; CHECK-NEXT: [[DOTSROA_18924_0:%.*]] = phi double [ [[TMP33]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ]
+; CHECK-NEXT: store double [[DOTSROA_0913_0]], ptr [[TMP1]], align 8
+; CHECK-NEXT: [[DOTSROA_6915_0__SROA_IDX916:%.*]] = getelementptr i8, ptr [[TMP1]], i64 8
+; CHECK-NEXT: store double [[DOTSROA_6915_0]], ptr [[DOTSROA_6915_0__SROA_IDX916]], align 8
+; CHECK-NEXT: [[DOTSROA_10918_0__SROA_IDX919:%.*]] = getelementptr i8, ptr [[TMP1]], i64 16
+; CHECK-NEXT: store double [[DOTSROA_10918_0]], ptr [[DOTSROA_10918_0__SROA_IDX919]], align 8
+; CHECK-NEXT: [[DOTSROA_14921_0__SROA_IDX922:%.*]] = getelementptr i8, ptr [[TMP1]], i64 24
+; CHECK-NEXT: store double [[DOTSROA_14921_0]], ptr [[DOTSROA_14921_0__SROA_IDX922]], align 8
+; CHECK-NEXT: store double [[DOTSROA_18924_0]], ptr [[TMP0]], align 8
+; CHECK-NEXT: ret void
+;
+ br i1 true, label %vector.ph, label %.preheader511
+
+vector.ph:
+ %5 = load double, ptr %0, align 8
+ %6 = fmul double %5, 0.000000e+00
+ %7 = fadd double %6, 0.000000e+00
+ %8 = fmul double %6, 0.000000e+00
+ %9 = fmul double %8, 0.000000e+00
+ %10 = fadd double %2, 0.000000e+00
+ %11 = fadd double %10, 1.000000e+00
+ %12 = fadd double %9, %11
+ %13 = fmul double %2, 0.000000e+00
+ %14 = fadd contract double %13, 0.000000e+00
+ %15 = fmul double %6, %2
+ %16 = fadd contract double %15, 0.000000e+00
+ %17 = load double, ptr %1, align 8
+ %18 = fmul double %17, 0.000000e+00
+ %19 = fmul double %18, %2
+ %20 = fmul double %2, %19
+ %21 = fadd double %20, 0.000000e+00
+ %22 = load double, ptr %0, align 8
+ %23 = fmul double %22, 0.000000e+00
+ %24 = fmul double %2, %23
+ %25 = fadd double %24, 0.000000e+00
+ %26 = fmul double %5, 0.000000e+00
+ %27 = fmul double %26, 0.000000e+00
+ %28 = fadd double %27, 0.000000e+00
+ %29 = fmul double %2, %17
+ %30 = fadd double %29, 0.000000e+00
+ br label %.preheader511
+
+.preheader511:
+ %.sroa.9.0 = phi double [ %30, %vector.ph ], [ 0.000000e+00, %4 ]
+ %.sroa.7.0 = phi double [ %28, %vector.ph ], [ 0.000000e+00, %4 ]
+ %.sroa.6887.0 = phi double [ %25, %vector.ph ], [ 0.000000e+00, %4 ]
+ %.sroa.5.0 = phi double [ %21, %vector.ph ], [ 0.000000e+00, %4 ]
+ %.sroa.4886.0 = phi double [ %16, %vector.ph ], [ 0.000000e+00, %4 ]
+ %.sroa.3.0 = phi double [ %14, %vector.ph ], [ 0.000000e+00, %4 ]
+ %.sroa.2885.0 = phi double [ %12, %vector.ph ], [ 0.000000e+00, %4 ]
+ %.sroa.0884.0 = phi double [ %7, %vector.ph ], [ 0.000000e+00, %4 ]
+ br i1 true, label %31, label %.preheader498
+
+31:
+ %32 = getelementptr i8, ptr %3, i64 8
+ %33 = load double, ptr %3, align 8
+ %34 = fmul double %.sroa.0884.0, %33
+ %35 = load double, ptr %32, align 8
+ %36 = fmul double %.sroa.2885.0, %35
+ %37 = fadd double %34, %36
+ %38 = fmul double %.sroa.4886.0, 0.000000e+00
+ %39 = fadd double %37, %38
+ %40 = fadd double %39, 0.000000e+00
+ %41 = fadd double %40, 1.000000e+00
+ %42 = fmul double %.sroa.2885.0, %33
+ %43 = fmul double %.sroa.3.0, %35
+ %44 = fadd double %42, %43
+ %45 = fmul double %.sroa.5.0, 0.000000e+00
+ %46 = fadd double %44, %45
+ %47 = fadd double %46, %2
+ %48 = fadd double %47, 0.000000e+00
+ %49 = fmul double %.sroa.4886.0, %33
+ %50 = fmul double %.sroa.5.0, %35
+ %51 = fadd double %49, %50
+ %52 = load double, ptr %0, align 8
+ %53 = fmul double %.sroa.6887.0, %52
+ %54 = fadd double %51, %53
+ %55 = fmul double %.sroa.9.0, %2
+ %56 = fadd double %54, %55
+ %57 = fadd double %56, 0.000000e+00
+ %58 = fmul double %.sroa.7.0, %33
+ %59 = fadd double %58, %2
+ %60 = fmul double %.sroa.9.0, 0.000000e+00
+ %61 = fadd double %59, %60
+ %62 = fadd double %61, 0.000000e+00
+ %63 = fadd double %62, 1.000000e+00
+ br label %.preheader498
+
+.preheader498:
+ %.sroa.0913.0 = phi double [ %41, %31 ], [ 0.000000e+00, %.preheader511 ]
+ %.sroa.6915.0 = phi double [ %48, %31 ], [ 0.000000e+00, %.preheader511 ]
+ %.sroa.10918.0 = phi double [ %57, %31 ], [ 0.000000e+00, %.preheader511 ]
+ %.sroa.14921.0 = phi double [ %63, %31 ], [ 0.000000e+00, %.preheader511 ]
+ %.sroa.18924.0 = phi double [ %33, %31 ], [ 0.000000e+00, %.preheader511 ]
+ store double %.sroa.0913.0, ptr %1, align 8
+ %.sroa.6915.0..sroa_idx916 = getelementptr i8, ptr %1, i64 8
+ store double %.sroa.6915.0, ptr %.sroa.6915.0..sroa_idx916, align 8
+ %.sroa.10918.0..sroa_idx919 = getelementptr i8, ptr %1, i64 16
+ store double %.sroa.10918.0, ptr %.sroa.10918.0..sroa_idx919, align 8
+ %.sroa.14921.0..sroa_idx922 = getelementptr i8, ptr %1, i64 24
+ store double %.sroa.14921.0, ptr %.sroa.14921.0..sroa_idx922, align 8
+ store double %.sroa.18924.0, ptr %0, align 8
+ ret void
+}
More information about the llvm-commits
mailing list