[llvm] [SLP]Fix unscheduled-deps assert for operands of copyable user lanes (PR #214283)

[via llvm-commits]

llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-llvm-transforms

Author: Alexey Bataev (alexey-bataev)

<details>
<summary>Changes</summary>

When every copyable model of an edge is skipped because the user is
itself a copyable lane, the instruction's own schedule data still
carries the def-use dependency counted by calculateDependencies. Release
it instead of returning early just because copyable data was found.

Fixes https://github.com/llvm/llvm-project/pull/213369#issuecomment-5189475412


---
Full diff: https://github.com/llvm/llvm-project/pull/214283.diff


2 Files Affected:

- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+6-2) 
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/fmuladd-absorbed-copyable-sched-deps.ll (+32) 


``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 229a225c6d413..3c14f2a0b595b 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4857,15 +4857,19 @@ class slpvectorizer::BoUpSLP {
           if (!ScheduleCopyableDataMap.empty()) {
             SmallVector<ScheduleCopyableData *> CopyableData =
                 getScheduleCopyableData(User, OpIdx, I);
+            bool ReleasedAsCopyable = false;
             for (ScheduleCopyableData *CD : CopyableData) {
               // Copyable elements modeled on a copyable user lane depend on
               // the user's copyable scheduling data, not on the user itself,
-              // and are released when that copyable data is scheduled.
+              // and are released when that copyable data is scheduled. The
+              // user's own schedule data still carries the def-use dependency
+              // in this case, so it must be released below.
               if (CD->getEdgeInfo().UserTE->isCopyableElement(User))
                 continue;
               DecrUnsched(CD, /*IsControl=*/false);
+              ReleasedAsCopyable = true;
             }
-            if (!CopyableData.empty())
+            if (ReleasedAsCopyable)
               return;
           }
           if (ScheduleData *OpSD = getScheduleData(I))
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fmuladd-absorbed-copyable-sched-deps.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fmuladd-absorbed-copyable-sched-deps.ll
index 128c25960d51a..217cc908065af 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/fmuladd-absorbed-copyable-sched-deps.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fmuladd-absorbed-copyable-sched-deps.ll
@@ -43,3 +43,35 @@ entry:
 declare double @llvm.fmuladd.f64(double, double, double) #0
 
 attributes #0 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
+
+; The operand of the absorbed copyable fmul is itself a copyable lane of the
+; child node; the def-use dependency of that operand on the absorbed fmul must
+; be released when the fmul is scheduled.
+define void @absorbed_fmul_copyable_operand() {
+; CHECK-LABEL: define void @absorbed_fmul_copyable_operand() {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr poison, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <4 x float> [[TMP0]], <float 1.000000e+00, float 0.000000e+00, float 1.000000e+00, float 1.000000e+00>
+; CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP1]], <4 x float> zeroinitializer, <4 x float> <float -0.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>)
+; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr poison, align 4
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load float, ptr poison, align 4
+  %mul60 = fmul float %0, 0.000000e+00
+  store float %mul60, ptr poison, align 4
+  %arrayidx65 = getelementptr i8, ptr poison, i64 4
+  %1 = load float, ptr %arrayidx65, align 4
+  %neg = fmul float %1, 0.000000e+00
+  %2 = call float @llvm.fmuladd.f32(float %neg, float 0.000000e+00, float 0.000000e+00)
+  store float %2, ptr %arrayidx65, align 4
+  %arrayidx65.1 = getelementptr i8, ptr poison, i64 8
+  %3 = load float, ptr %arrayidx65.1, align 4
+  %4 = call float @llvm.fmuladd.f32(float %3, float 0.000000e+00, float 0.000000e+00)
+  store float %4, ptr %arrayidx65.1, align 4
+  %arrayidx65.2 = getelementptr i8, ptr poison, i64 12
+  %5 = load float, ptr %arrayidx65.2, align 4
+  %6 = call float @llvm.fmuladd.f32(float %5, float 0.000000e+00, float 0.000000e+00)
+  store float %6, ptr %arrayidx65.2, align 4
+  ret void
+}

``````````

</details>


https://github.com/llvm/llvm-project/pull/214283