[llvm] 9c86198 - [SLP] Update vector value for incoming phi node, beeing vectorized already
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri Mar 14 12:54:08 PDT 2025
Author: Alexey Bataev
Date: 2025-03-14T12:53:56-07:00
New Revision: 9c86198caf3654a744ae3fb15447040153acba19
URL: https://github.com/llvm/llvm-project/commit/9c86198caf3654a744ae3fb15447040153acba19
DIFF: https://github.com/llvm/llvm-project/commit/9c86198caf3654a744ae3fb15447040153acba19.diff
LOG: [SLP] Update vector value for incoming phi node, beeing vectorized already
If the phi node contains multiple same incoming blocks/values, need to
update the corresponding vectorized value, if it is not going to be
vectorized, if the incoming value was vectorized already.
Fixes #131355
Added:
llvm/test/Transforms/SLPVectorizer/X86/multi-incoming-blocks-in-phi.ll
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 184d5bf4a86c0..9c3992b4a713d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3084,6 +3084,10 @@ class BoUpSLP {
/// \returns the graph entry for the \p Idx operand of the \p E entry.
const TreeEntry *getOperandEntry(const TreeEntry *E, unsigned Idx) const;
+ TreeEntry *getOperandEntry(TreeEntry *E, unsigned Idx) {
+ return const_cast<TreeEntry *>(
+ getOperandEntry(const_cast<const TreeEntry *>(E), Idx));
+ }
/// Gets the root instruction for the given node. If the node is a strided
/// load/store node with the reverse order, the root instruction is the last
@@ -10759,7 +10763,7 @@ void BoUpSLP::transformNodes() {
break;
// This node is a minmax node.
E.CombinedOp = TreeEntry::MinMax;
- TreeEntry *CondEntry = const_cast<TreeEntry *>(getOperandEntry(&E, 0));
+ TreeEntry *CondEntry = getOperandEntry(&E, 0);
if (SelectOnly && CondEntry->UserTreeIndex &&
CondEntry->State == TreeEntry::Vectorize) {
// The condition node is part of the combined minmax node.
@@ -16376,7 +16380,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
// visit every block once.
SmallPtrSet<BasicBlock *, 4> VisitedBBs;
- for (unsigned I : seq<unsigned>(0, PH->getNumIncomingValues())) {
+ for (unsigned I : seq<unsigned>(PH->getNumIncomingValues())) {
ValueList Operands;
BasicBlock *IBB = PH->getIncomingBlock(I);
@@ -16387,7 +16391,10 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
}
if (!VisitedBBs.insert(IBB).second) {
- NewPhi->addIncoming(NewPhi->getIncomingValueForBlock(IBB), IBB);
+ Value *VecOp = NewPhi->getIncomingValueForBlock(IBB);
+ NewPhi->addIncoming(VecOp, IBB);
+ TreeEntry *OpTE = getOperandEntry(E, I);
+ OpTE->VectorizedValue = VecOp;
continue;
}
@@ -18965,7 +18972,7 @@ bool BoUpSLP::collectValuesToDemote(
const unsigned NumOps = E.getNumOperands();
SmallVector<const TreeEntry *> Ops(NumOps);
transform(seq<unsigned>(0, NumOps), Ops.begin(),
- std::bind(&BoUpSLP::getOperandEntry, this, &E, _1));
+ [&](unsigned Idx) { return getOperandEntry(&E, Idx); });
return TryProcessInstruction(BitWidth, Ops);
}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi-incoming-blocks-in-phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi-incoming-blocks-in-phi.ll
new file mode 100644
index 0000000000000..ba35de391f9eb
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi-incoming-blocks-in-phi.ll
@@ -0,0 +1,189 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=znver2 < %s | FileCheck %s
+
+define void @foo(ptr %arg) {
+; CHECK-LABEL: define void @foo(
+; CHECK-SAME: ptr [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[CALL:%.*]] = call noalias ptr null(i32 9, i64 16, i64 816, i64 400, i64 0)
+; CHECK-NEXT: [[ICMP:%.*]] = icmp eq ptr [[CALL]], null
+; CHECK-NEXT: [[ADD:%.*]] = add <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, splat (i16 16)
+; CHECK-NEXT: [[AND:%.*]] = and <16 x i64> zeroinitializer, splat (i64 1)
+; CHECK-NEXT: [[ADD1:%.*]] = add nuw nsw i64 0, 48
+; CHECK-NEXT: [[SELECT:%.*]] = select <16 x i1> zeroinitializer, <16 x i16> zeroinitializer, <16 x i16> zeroinitializer
+; CHECK-NEXT: [[ADD2:%.*]] = add nsw <16 x i16> [[SELECT]], splat (i16 -6124)
+; CHECK-NEXT: [[ADD3:%.*]] = add <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, splat (i16 64)
+; CHECK-NEXT: [[SUB:%.*]] = sub nsw <16 x i16> zeroinitializer, [[ADD3]]
+; CHECK-NEXT: [[SELECT4:%.*]] = select <16 x i1> zeroinitializer, <16 x i16> [[ADD3]], <16 x i16> [[SUB]]
+; CHECK-NEXT: [[ADD5:%.*]] = add <16 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, splat (i64 80)
+; CHECK-NEXT: [[FADD:%.*]] = fadd float 0.000000e+00, -1.580000e+02
+; CHECK-NEXT: store float 0.000000e+00, ptr null, align 4
+; CHECK-NEXT: [[ICMP6:%.*]] = icmp eq i32 0, 0
+; CHECK-NEXT: call void null()
+; CHECK-NEXT: [[MUL:%.*]] = mul i32 0, 9
+; CHECK-NEXT: br label %[[BB7:.*]]
+; CHECK: [[BB7]]:
+; CHECK-NEXT: [[PHI:%.*]] = phi float [ 4.000000e+00, %[[BB]] ], [ 0.000000e+00, %[[BB27:.*]] ]
+; CHECK-NEXT: [[FADD8:%.*]] = fadd float 0.000000e+00, 0.000000e+00
+; CHECK-NEXT: [[FADD9:%.*]] = fadd float [[PHI]], 1.000000e+00
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> <float poison, float 0.000000e+00>, float [[FADD9]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> <float 1.000000e+00, float 0.000000e+00>, [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[FREM:%.*]] = frem float [[TMP2]], 7.000000e+00
+; CHECK-NEXT: [[CALL12:%.*]] = call i32 @llvm.x86.sse.cvttss2si(<4 x float> zeroinitializer)
+; CHECK-NEXT: switch i32 [[CALL12]], label %[[BB13:.*]] [
+; CHECK-NEXT: i32 125, label %[[BB30:.*]]
+; CHECK-NEXT: i32 98, label %[[BB30]]
+; CHECK-NEXT: i32 99, label %[[BB30]]
+; CHECK-NEXT: i32 103, label %[[BB30]]
+; CHECK-NEXT: i32 112, label %[[BB30]]
+; CHECK-NEXT: i32 116, label %[[BB30]]
+; CHECK-NEXT: i32 121, label %[[BB30]]
+; CHECK-NEXT: ]
+; CHECK: [[BB13]]:
+; CHECK-NEXT: [[FMUL:%.*]] = fmul double 0.000000e+00, 1.220000e+02
+; CHECK-NEXT: [[MUL14:%.*]] = mul i32 0, -3
+; CHECK-NEXT: [[MUL15:%.*]] = mul i32 [[MUL14]], -3
+; CHECK-NEXT: [[CALL16:%.*]] = call i32 @llvm.x86.sse.cvttss2si(<4 x float> zeroinitializer)
+; CHECK-NEXT: switch i32 [[CALL16]], label %[[BB17:.*]] [
+; CHECK-NEXT: i32 125, label %[[BB30]]
+; CHECK-NEXT: i32 98, label %[[BB30]]
+; CHECK-NEXT: i32 99, label %[[BB30]]
+; CHECK-NEXT: i32 103, label %[[BB30]]
+; CHECK-NEXT: i32 112, label %[[BB30]]
+; CHECK-NEXT: i32 116, label %[[BB30]]
+; CHECK-NEXT: i32 121, label %[[BB30]]
+; CHECK-NEXT: ]
+; CHECK: [[BB17]]:
+; CHECK-NEXT: [[FADD18:%.*]] = fadd float 0.000000e+00, 1.000000e+00
+; CHECK-NEXT: [[MUL19:%.*]] = mul i32 [[MUL15]], -3
+; CHECK-NEXT: br label %[[BB20:.*]]
+; CHECK: [[BB20]]:
+; CHECK-NEXT: [[FADD21:%.*]] = fadd float [[FADD18]], 1.000000e+00
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> <float poison, float 0.000000e+00>, float [[FADD21]], i32 0
+; CHECK-NEXT: switch i32 0, label %[[BB22:.*]] [
+; CHECK-NEXT: i32 125, label %[[BB30]]
+; CHECK-NEXT: i32 98, label %[[BB30]]
+; CHECK-NEXT: i32 99, label %[[BB30]]
+; CHECK-NEXT: i32 103, label %[[BB30]]
+; CHECK-NEXT: i32 112, label %[[BB30]]
+; CHECK-NEXT: i32 116, label %[[BB30]]
+; CHECK-NEXT: i32 121, label %[[BB30]]
+; CHECK-NEXT: ]
+; CHECK: [[BB22]]:
+; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x float> <float 1.000000e+00, float 0.000000e+00>, [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP4]], i32 0
+; CHECK-NEXT: [[FREM25:%.*]] = frem float [[TMP5]], 7.000000e+00
+; CHECK-NEXT: [[FMUL26:%.*]] = fmul float [[FREM25]], 5.000000e+00
+; CHECK-NEXT: switch i32 0, label %[[BB27]] [
+; CHECK-NEXT: i32 125, label %[[BB30]]
+; CHECK-NEXT: i32 98, label %[[BB30]]
+; CHECK-NEXT: i32 99, label %[[BB30]]
+; CHECK-NEXT: i32 103, label %[[BB30]]
+; CHECK-NEXT: i32 112, label %[[BB30]]
+; CHECK-NEXT: i32 116, label %[[BB30]]
+; CHECK-NEXT: i32 121, label %[[BB30]]
+; CHECK-NEXT: ]
+; CHECK: [[BB27]]:
+; CHECK-NEXT: [[FADD28:%.*]] = fadd float [[TMP5]], 1.000000e+00
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x float> [[TMP4]], i32 1
+; CHECK-NEXT: [[FADD29:%.*]] = fadd float [[TMP6]], 0.000000e+00
+; CHECK-NEXT: br label %[[BB7]]
+; CHECK: [[BB30]]:
+; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x float> [ [[TMP1]], %[[BB7]] ], [ [[TMP1]], %[[BB7]] ], [ [[TMP1]], %[[BB7]] ], [ [[TMP1]], %[[BB7]] ], [ [[TMP1]], %[[BB7]] ], [ [[TMP1]], %[[BB7]] ], [ [[TMP1]], %[[BB7]] ], [ zeroinitializer, %[[BB13]] ], [ zeroinitializer, %[[BB13]] ], [ zeroinitializer, %[[BB13]] ], [ zeroinitializer, %[[BB13]] ], [ zeroinitializer, %[[BB13]] ], [ zeroinitializer, %[[BB13]] ], [ zeroinitializer, %[[BB13]] ], [ [[TMP3]], %[[BB20]] ], [ [[TMP3]], %[[BB20]] ], [ [[TMP3]], %[[BB20]] ], [ [[TMP3]], %[[BB20]] ], [ [[TMP3]], %[[BB20]] ], [ [[TMP3]], %[[BB20]] ], [ [[TMP3]], %[[BB20]] ], [ [[TMP4]], %[[BB22]] ], [ [[TMP4]], %[[BB22]] ], [ [[TMP4]], %[[BB22]] ], [ [[TMP4]], %[[BB22]] ], [ [[TMP4]], %[[BB22]] ], [ [[TMP4]], %[[BB22]] ], [ [[TMP4]], %[[BB22]] ]
+; CHECK-NEXT: ret void
+;
+bb:
+ %call = call noalias ptr null(i32 9, i64 16, i64 816, i64 400, i64 0)
+ %icmp = icmp eq ptr %call, null
+ %add = add <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, splat (i16 16)
+ %and = and <16 x i64> zeroinitializer, splat (i64 1)
+ %add1 = add nuw nsw i64 0, 48
+ %select = select <16 x i1> zeroinitializer, <16 x i16> zeroinitializer, <16 x i16> zeroinitializer
+ %add2 = add nsw <16 x i16> %select, splat (i16 -6124)
+ %add3 = add <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, splat (i16 64)
+ %sub = sub nsw <16 x i16> zeroinitializer, %add3
+ %select4 = select <16 x i1> zeroinitializer, <16 x i16> %add3, <16 x i16> %sub
+ %add5 = add <16 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>, splat (i64 80)
+ %fadd = fadd float 0.000000e+00, -1.580000e+02
+ store float 0.000000e+00, ptr null, align 4
+ %icmp6 = icmp eq i32 0, 0
+ call void null()
+ %mul = mul i32 0, 9
+ br label %bb7
+
+bb7:
+ %phi = phi float [ 4.000000e+00, %bb ], [ 0.000000e+00, %bb27 ]
+ %fadd8 = fadd float 0.000000e+00, 0.000000e+00
+ %fadd9 = fadd float %phi, 1.000000e+00
+ %fadd10 = fadd float %fadd9, 1.000000e+00
+ %fadd11 = fadd float 0.000000e+00, 0.000000e+00
+ %frem = frem float %fadd10, 7.000000e+00
+ %call12 = call i32 @llvm.x86.sse.cvttss2si(<4 x float> zeroinitializer)
+ switch i32 %call12, label %bb13 [
+ i32 125, label %bb30
+ i32 98, label %bb30
+ i32 99, label %bb30
+ i32 103, label %bb30
+ i32 112, label %bb30
+ i32 116, label %bb30
+ i32 121, label %bb30
+ ]
+
+bb13:
+ %fmul = fmul double 0.000000e+00, 1.220000e+02
+ %mul14 = mul i32 0, -3
+ %mul15 = mul i32 %mul14, -3
+ %call16 = call i32 @llvm.x86.sse.cvttss2si(<4 x float> zeroinitializer)
+ switch i32 %call16, label %bb17 [
+ i32 125, label %bb30
+ i32 98, label %bb30
+ i32 99, label %bb30
+ i32 103, label %bb30
+ i32 112, label %bb30
+ i32 116, label %bb30
+ i32 121, label %bb30
+ ]
+
+bb17:
+ %fadd18 = fadd float 0.000000e+00, 1.000000e+00
+ %mul19 = mul i32 %mul15, -3
+ br label %bb20
+
+bb20:
+ %fadd21 = fadd float %fadd18, 1.000000e+00
+ switch i32 0, label %bb22 [
+ i32 125, label %bb30
+ i32 98, label %bb30
+ i32 99, label %bb30
+ i32 103, label %bb30
+ i32 112, label %bb30
+ i32 116, label %bb30
+ i32 121, label %bb30
+ ]
+
+bb22:
+ %fadd23 = fadd float %fadd21, 1.000000e+00
+ %fadd24 = fadd float 0.000000e+00, 0.000000e+00
+ %frem25 = frem float %fadd23, 7.000000e+00
+ %fmul26 = fmul float %frem25, 5.000000e+00
+ switch i32 0, label %bb27 [
+ i32 125, label %bb30
+ i32 98, label %bb30
+ i32 99, label %bb30
+ i32 103, label %bb30
+ i32 112, label %bb30
+ i32 116, label %bb30
+ i32 121, label %bb30
+ ]
+
+bb27:
+ %fadd28 = fadd float %fadd23, 1.000000e+00
+ %fadd29 = fadd float %fadd24, 0.000000e+00
+ br label %bb7
+
+bb30:
+ %phi31 = phi float [ %fadd10, %bb7 ], [ %fadd10, %bb7 ], [ %fadd10, %bb7 ], [ %fadd10, %bb7 ], [ %fadd10, %bb7 ], [ %fadd10, %bb7 ], [ %fadd10, %bb7 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ %fadd21, %bb20 ], [ %fadd21, %bb20 ], [ %fadd21, %bb20 ], [ %fadd21, %bb20 ], [ %fadd21, %bb20 ], [ %fadd21, %bb20 ], [ %fadd21, %bb20 ], [ %fadd23, %bb22 ], [ %fadd23, %bb22 ], [ %fadd23, %bb22 ], [ %fadd23, %bb22 ], [ %fadd23, %bb22 ], [ %fadd23, %bb22 ], [ %fadd23, %bb22 ]
+ %phi32 = phi float [ %fadd11, %bb7 ], [ %fadd11, %bb7 ], [ %fadd11, %bb7 ], [ %fadd11, %bb7 ], [ %fadd11, %bb7 ], [ %fadd11, %bb7 ], [ %fadd11, %bb7 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb13 ], [ 0.000000e+00, %bb20 ], [ 0.000000e+00, %bb20 ], [ 0.000000e+00, %bb20 ], [ 0.000000e+00, %bb20 ], [ 0.000000e+00, %bb20 ], [ 0.000000e+00, %bb20 ], [ 0.000000e+00, %bb20 ], [ %fadd24, %bb22 ], [ %fadd24, %bb22 ], [ %fadd24, %bb22 ], [ %fadd24, %bb22 ], [ %fadd24, %bb22 ], [ %fadd24, %bb22 ], [ %fadd24, %bb22 ]
+ ret void
+}
More information about the llvm-commits
mailing list