[llvm] [SLP]Fix unscheduled-deps assert for copyable operands in reassociated nodes (PR #215016)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 8 13:05:13 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
For a non-commutative user in a reassociated (flattened) node, the
copyable operand may be modeled on a flattened operand column rather
than at the instruction's operand number, so the direct lookup misses
it and the def-use dependency is counted but never released. Scan the
flattened operand columns at the user's lane for the copyable data.
---
Full diff: https://github.com/llvm/llvm-project/pull/215016.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+21-1)
- (added) llvm/test/Transforms/SLPVectorizer/X86/reassoc-flattened-copyable-operand.ll (+99)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 444c5937f15b2..fea4f23df7668 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4633,8 +4633,28 @@ class slpvectorizer::BoUpSLP {
IsCommutativeUser && User->getOperand(0) == User->getOperand(1);
if ((!IsCommutativeUser || IsCommutativeWithSameOps) &&
!isa<CmpInst>(User)) {
+ if (CurNumOps != NumOps)
+ continue;
+ // A reassociated node flattens the operand chain, so the operand
+ // may be placed in any operand column rather than at the
+ // instruction's operand number.
+ if (TE->hasReassocScalars()) {
+ bool ReplacedByCopyable = false;
+ for (auto It = find(TE->Scalars, User); It != TE->Scalars.end();
+ It = find(make_range(std::next(It), TE->Scalars.end()),
+ User)) {
+ int Lane = std::distance(TE->Scalars.begin(), It);
+ for (unsigned OpIdx : seq<unsigned>(TE->getNumOperands()))
+ ReplacedByCopyable |=
+ TE->getOperand(OpIdx)[Lane] == Op &&
+ getScheduleCopyableData(EdgeInfo(TE, OpIdx), Op);
+ }
+ if (ReplacedByCopyable)
+ continue;
+ return false;
+ }
EdgeInfo EI(TE, U.getOperandNo());
- if (CurNumOps != NumOps || getScheduleCopyableData(EI, Op))
+ if (getScheduleCopyableData(EI, Op))
continue;
return false;
}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reassoc-flattened-copyable-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/reassoc-flattened-copyable-operand.ll
new file mode 100644
index 0000000000000..91368f5b80ead
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reassoc-flattened-copyable-operand.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=znver4 | FileCheck %s
+
+ at hexsh_ = external global [4496 x i8]
+
+define void @test(ptr %0, ptr %1, ptr %2, ptr %3, ptr %4, ptr %5, ptr %6, ptr %7, ptr %_QMshared_common_dataEp2th, double %8, double %9, double %10, double %11, double %12, ptr %_QMshared_common_dataEpone, double %13, double %14, double %15, ptr %16, double %17, double %18, double %19, double %20, double %21, double %22, double %23, double %24, double %25, ptr %_QMshared_common_dataEptwo, ptr %_QMshared_common_dataEpthr, ptr %_QMshared_common_dataEp72th, ptr %26, double %27, double %28, double %29, double %30, double %31, double %32, double %33, double %34, double %35, double %36, double %37, double %38, double %39, double %40, double %41, double %42, double %43, double %44, double %45, double %46, double %47, double %48) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[TMP1:%.*]], ptr [[TMP2:%.*]], ptr [[TMP3:%.*]], ptr [[TMP4:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]], ptr [[TMP7:%.*]], ptr [[_QMSHARED_COMMON_DATAEP2TH:%.*]], double [[TMP8:%.*]], double [[TMP9:%.*]], double [[TMP10:%.*]], double [[TMP11:%.*]], double [[TMP12:%.*]], ptr [[_QMSHARED_COMMON_DATAEPONE:%.*]], double [[TMP13:%.*]], double [[TMP14:%.*]], double [[TMP15:%.*]], ptr [[TMP16:%.*]], double [[TMP17:%.*]], double [[TMP18:%.*]], double [[TMP19:%.*]], double [[TMP20:%.*]], double [[TMP21:%.*]], double [[TMP22:%.*]], double [[TMP23:%.*]], double [[TMP24:%.*]], double [[TMP25:%.*]], ptr [[_QMSHARED_COMMON_DATAEPTWO:%.*]], ptr [[_QMSHARED_COMMON_DATAEPTHR:%.*]], ptr [[_QMSHARED_COMMON_DATAEP72TH:%.*]], ptr [[TMP26:%.*]], double [[TMP27:%.*]], double [[TMP28:%.*]], double [[TMP29:%.*]], double [[TMP30:%.*]], double [[TMP31:%.*]], double [[TMP32:%.*]], double [[TMP33:%.*]], double [[TMP34:%.*]], double [[TMP35:%.*]], double [[TMP36:%.*]], double [[TMP37:%.*]], double [[TMP38:%.*]], double [[TMP39:%.*]], double [[TMP40:%.*]], double [[TMP41:%.*]], double [[TMP42:%.*]], double [[TMP43:%.*]], double [[TMP44:%.*]], double [[TMP45:%.*]], double [[TMP46:%.*]], double [[TMP47:%.*]], double [[TMP48:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[_PREHEADER4179_PREHEADER:.*:]]
+; CHECK-NEXT: store double 1.000000e+00, ptr [[TMP26]], align 8
+; CHECK-NEXT: [[TMP49:%.*]] = fadd fast double [[TMP38]], 1.000000e+00
+; CHECK-NEXT: [[TMP50:%.*]] = fadd fast double [[TMP42]], 1.000000e+00
+; CHECK-NEXT: store double [[TMP44]], ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 224), align 8
+; CHECK-NEXT: [[TMP51:%.*]] = fmul fast double [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP52:%.*]] = fmul fast double [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP53:%.*]] = fneg fast double [[TMP25]]
+; CHECK-NEXT: [[TMP54:%.*]] = fadd fast double [[TMP51]], 1.000000e+00
+; CHECK-NEXT: [[TMP55:%.*]] = fmul fast double [[TMP48]], [[TMP40]]
+; CHECK-NEXT: [[TMP56:%.*]] = fmul fast double [[TMP36]], [[TMP46]]
+; CHECK-NEXT: [[TMP57:%.*]] = fmul fast double [[TMP32]], [[TMP40]]
+; CHECK-NEXT: [[TMP58:%.*]] = fmul fast double [[TMP39]], [[TMP40]]
+; CHECK-NEXT: [[TMP59:%.*]] = fadd fast double [[TMP56]], 1.000000e+00
+; CHECK-NEXT: [[TMP60:%.*]] = fmul fast double [[TMP24]], [[TMP53]]
+; CHECK-NEXT: [[TMP61:%.*]] = fmul fast double [[TMP13]], [[TMP51]]
+; CHECK-NEXT: [[TMP62:%.*]] = fmul fast double [[TMP13]], [[TMP52]]
+; CHECK-NEXT: [[TMP63:%.*]] = fmul fast double [[TMP36]], [[TMP62]]
+; CHECK-NEXT: [[TMP64:%.*]] = fmul fast double [[TMP36]], [[TMP61]]
+; CHECK-NEXT: [[TMP65:%.*]] = fmul fast double [[TMP45]], [[TMP62]]
+; CHECK-NEXT: [[TMP66:%.*]] = insertelement <2 x double> poison, double [[TMP65]], i64 0
+; CHECK-NEXT: [[TMP67:%.*]] = insertelement <2 x double> [[TMP66]], double [[TMP64]], i64 1
+; CHECK-NEXT: [[TMP68:%.*]] = fadd fast <2 x double> [[TMP67]], <double -0.000000e+00, double 2.000000e+00>
+; CHECK-NEXT: [[TMP69:%.*]] = fsub fast double [[TMP63]], [[TMP59]]
+; CHECK-NEXT: [[TMP70:%.*]] = insertelement <4 x double> poison, double [[TMP69]], i64 0
+; CHECK-NEXT: [[TMP71:%.*]] = shufflevector <2 x double> [[TMP68]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP72:%.*]] = shufflevector <4 x double> [[TMP70]], <4 x double> [[TMP71]], <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
+; CHECK-NEXT: [[TMP73:%.*]] = insertelement <4 x double> [[TMP72]], double [[TMP60]], i64 3
+; CHECK-NEXT: [[TMP74:%.*]] = insertelement <4 x double> poison, double [[TMP47]], i64 0
+; CHECK-NEXT: [[TMP75:%.*]] = insertelement <4 x double> [[TMP74]], double [[TMP55]], i64 1
+; CHECK-NEXT: [[TMP76:%.*]] = insertelement <4 x double> [[TMP75]], double [[TMP62]], i64 2
+; CHECK-NEXT: [[TMP77:%.*]] = insertelement <4 x double> [[TMP76]], double [[TMP54]], i64 3
+; CHECK-NEXT: [[TMP78:%.*]] = fadd fast <4 x double> [[TMP73]], [[TMP77]]
+; CHECK-NEXT: [[TMP79:%.*]] = fsub fast <4 x double> [[TMP73]], [[TMP77]]
+; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <4 x double> [[TMP78]], <4 x double> [[TMP79]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP81:%.*]] = fsub fast double [[TMP50]], [[TMP57]]
+; CHECK-NEXT: [[TMP82:%.*]] = fsub fast double [[TMP49]], [[TMP58]]
+; CHECK-NEXT: [[TMP83:%.*]] = insertelement <8 x double> poison, double [[TMP35]], i64 0
+; CHECK-NEXT: [[TMP84:%.*]] = insertelement <8 x double> [[TMP83]], double [[TMP82]], i64 1
+; CHECK-NEXT: [[TMP85:%.*]] = insertelement <8 x double> [[TMP84]], double [[TMP81]], i64 2
+; CHECK-NEXT: [[TMP86:%.*]] = insertelement <8 x double> [[TMP85]], double [[TMP61]], i64 3
+; CHECK-NEXT: [[TMP87:%.*]] = shufflevector <4 x double> [[TMP80]], <4 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP88:%.*]] = shufflevector <8 x double> [[TMP86]], <8 x double> [[TMP87]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP89:%.*]] = insertelement <8 x double> <double 1.000000e+00, double poison, double 1.000000e+00, double poison, double 1.000000e+00, double 1.000000e+00, double poison, double poison>, double [[TMP27]], i64 1
+; CHECK-NEXT: [[TMP90:%.*]] = shufflevector <8 x double> [[TMP89]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 1, i32 4, i32 5, i32 1, i32 1>
+; CHECK-NEXT: [[TMP91:%.*]] = fmul reassoc nsz arcp contract afn <8 x double> [[TMP88]], [[TMP90]]
+; CHECK-NEXT: store <8 x double> [[TMP91]], ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 136), align 8
+; CHECK-NEXT: ret void
+;
+.preheader4179.preheader:
+ %49 = fmul fast double %8, %9
+ %50 = fmul fast double %8, %10
+ %51 = fmul fast double %13, %49
+ %52 = fmul fast double %13, %50
+ %53 = fneg fast double %25
+ %54 = fmul fast double %24, %53
+ store double 1.000000e+00, ptr %26, align 8
+ %55 = fadd fast double %50, 1.000000e+00
+ %56 = fsub fast double %54, %55
+ %57 = fmul fast double %56, %27
+ store double %57, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 192), align 8
+ store double %35, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 136), align 8
+ %58 = fadd fast double %38, 1.000000e+00
+ %59 = fmul fast double %39, %40
+ %60 = fsub fast double %58, %59
+ %61 = fmul fast double %60, %27
+ store double %61, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 144), align 8
+ %62 = fadd fast double %42, 1.000000e+00
+ %63 = fmul fast double %32, %40
+ %64 = fsub fast double %62, %63
+ store double %64, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 152), align 8
+ store double %44, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 224), align 8
+ %65 = fmul fast double %45, %51
+ %66 = fmul fast double %52, %27
+ store double %66, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 160), align 8
+ %67 = fmul fast double %36, %46
+ %68 = fmul fast double %36, %51
+ %69 = fadd fast double %67, 1.000000e+00
+ %70 = fsub fast double %68, %69
+ %71 = fadd fast double %70, %47
+ store double %71, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 168), align 8
+ %72 = fmul fast double %48, %40
+ %73 = fsub fast double %65, %72
+ store double %73, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 176), align 8
+ %74 = fmul fast double %36, %52
+ %75 = fadd fast double %74, 2.000000e+00
+ %76 = fsub fast double %75, %51
+ %77 = fmul fast double %76, %27
+ store double %77, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 184), align 8
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/215016
More information about the llvm-commits
mailing list