[llvm] d81ffd4 - [SLP]INsert postponed vector value after all uses, if the parent node is PHI
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 12 13:41:17 PDT 2025
Author: Alexey Bataev
Date: 2025-10-12T13:41:08-07:00
New Revision: d81ffd4ebb45235b4d106f6a0d5e5032bad41018
URL: https://github.com/llvm/llvm-project/commit/d81ffd4ebb45235b4d106f6a0d5e5032bad41018
DIFF: https://github.com/llvm/llvm-project/commit/d81ffd4ebb45235b4d106f6a0d5e5032bad41018.diff
LOG: [SLP]INsert postponed vector value after all uses, if the parent node is PHI
Need to insert the vector value for the postponed gather/buildvector
node after all uses non only if the vector value of the user node is
phi, but also if the user node itself is PHI node, which may produce
vector phi + shuffle.
Fixes #162799
Added:
llvm/test/Transforms/SLPVectorizer/X86/parent-phi-node-reordered.ll
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/X86/phi-nodes-incoming-same-blocks.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 96f05fc82f296..a6f4beca88058 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -20076,7 +20076,9 @@ Value *BoUpSLP::vectorizeTree(
// The is because source vector that supposed to feed this gather node was
// inserted at the end of the block [after stab instruction]. So we need
// to adjust insertion point again to the end of block.
- if (isa<PHINode>(UserI)) {
+ if (isa<PHINode>(UserI) ||
+ (TE->UserTreeIndex.UserTE->hasState() &&
+ TE->UserTreeIndex.UserTE->getOpcode() == Instruction::PHI)) {
// Insert before all users.
Instruction *InsertPt = PrevVec->getParent()->getTerminator();
for (User *U : PrevVec->users()) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/parent-phi-node-reordered.ll b/llvm/test/Transforms/SLPVectorizer/X86/parent-phi-node-reordered.ll
new file mode 100644
index 0000000000000..d01c35f6bc090
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/parent-phi-node-reordered.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -slp-threshold=-99999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define void @test(i32 %arg, i32 %arg1) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[ARG1:%.*]]) {
+; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> poison, i32 [[ARG1]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[ARG]], i32 0
+; CHECK-NEXT: br label %[[BB6:.*]]
+; CHECK: [[BB2:.*]]:
+; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ [[TMP14:%.*]], %[[BB19:.*]] ]
+; CHECK-NEXT: ret void
+; CHECK: [[BB6]]:
+; CHECK-NEXT: [[TMP3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP17:%.*]], %[[BB26:.*]] ], [ [[TMP16:%.*]], %[[BB27:.*]] ], [ zeroinitializer, %[[BB25:.*]] ]
+; CHECK-NEXT: switch i8 0, label %[[BB11:.*]] [
+; CHECK-NEXT: i8 0, label %[[BB28:.*]]
+; CHECK-NEXT: ]
+; CHECK: [[BB11]]:
+; CHECK-NEXT: [[PHI12:%.*]] = phi i32 [ 0, %[[BB28]] ], [ 0, %[[BB6]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = phi <4 x i32> [ [[TMP3]], %[[BB28]] ], [ zeroinitializer, %[[BB6]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> <i32 poison, i32 0, i32 poison, i32 poison>, <4 x i32> <i32 0, i32 5, i32 2, i32 3>
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[ARG]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[PHI12]], i32 3
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> <i32 poison, i32 0, i32 poison, i32 0>, <4 x i32> <i32 poison, i32 5, i32 2, i32 7>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP1]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; CHECK-NEXT: switch i8 0, label %[[BB19]] [
+; CHECK-NEXT: i8 1, label %[[BB17:.*]]
+; CHECK-NEXT: i8 0, label %[[BB18:.*]]
+; CHECK-NEXT: ]
+; CHECK: [[BB17]]:
+; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP10]], <4 x i32> <i32 0, i32 3, i32 6, i32 poison>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> <i32 poison, i32 poison, i32 poison, i32 0>, <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT: br label %[[BB19]]
+; CHECK: [[BB18]]:
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 2, i32 0>
+; CHECK-NEXT: br label %[[BB19]]
+; CHECK: [[BB19]]:
+; CHECK-NEXT: [[TMP14]] = phi <4 x i32> [ [[TMP10]], %[[BB17]] ], [ [[TMP7]], %[[BB18]] ], [ [[TMP9]], %[[BB11]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = phi <4 x i32> [ [[TMP12]], %[[BB17]] ], [ [[TMP13]], %[[BB18]] ], [ [[TMP7]], %[[BB11]] ]
+; CHECK-NEXT: [[TMP16]] = shufflevector <4 x i32> [[TMP15]], <4 x i32> poison, <4 x i32> <i32 0, i32 3, i32 2, i32 1>
+; CHECK-NEXT: br i1 false, label %[[BB2]], label %[[BB25]]
+; CHECK: [[BB25]]:
+; CHECK-NEXT: switch i8 0, label %[[BB6]] [
+; CHECK-NEXT: i8 0, label %[[BB26]]
+; CHECK-NEXT: i8 1, label %[[BB27]]
+; CHECK-NEXT: i8 6, label %[[BB27]]
+; CHECK-NEXT: ]
+; CHECK: [[BB26]]:
+; CHECK-NEXT: [[TMP17]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> [[TMP0]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; CHECK-NEXT: br label %[[BB6]]
+; CHECK: [[BB27]]:
+; CHECK-NEXT: br label %[[BB6]]
+; CHECK: [[BB28]]:
+; CHECK-NEXT: br label %[[BB11]]
+;
+bb:
+ br label %bb6
+
+bb2:
+ %phi = phi i32 [ %phi21, %bb19 ]
+ %phi3 = phi i32 [ %phi22, %bb19 ]
+ %phi4 = phi i32 [ %phi23, %bb19 ]
+ %phi5 = phi i32 [ %phi24, %bb19 ]
+ ret void
+
+bb6:
+ %phi7 = phi i32 [ 0, %bb ], [ %phi24, %bb26 ], [ %phi24, %bb27 ], [ 0, %bb25 ]
+ %phi8 = phi i32 [ 0, %bb ], [ %arg1, %bb26 ], [ %phi23, %bb27 ], [ 0, %bb25 ]
+ %phi9 = phi i32 [ 0, %bb ], [ %phi22, %bb26 ], [ %phi20, %bb27 ], [ 0, %bb25 ]
+ %phi10 = phi i32 [ 0, %bb ], [ %phi21, %bb26 ], [ %phi21, %bb27 ], [ 0, %bb25 ]
+ switch i8 0, label %bb11 [
+ i8 0, label %bb28
+ ]
+
+bb11:
+ %phi12 = phi i32 [ 0, %bb28 ], [ 0, %bb6 ]
+ %phi13 = phi i32 [ %phi10, %bb28 ], [ 0, %bb6 ]
+ %phi14 = phi i32 [ %phi9, %bb28 ], [ 0, %bb6 ]
+ %phi15 = phi i32 [ %phi8, %bb28 ], [ 0, %bb6 ]
+ %phi16 = phi i32 [ %phi7, %bb28 ], [ 0, %bb6 ]
+ switch i8 0, label %bb19 [
+ i8 1, label %bb17
+ i8 0, label %bb18
+ ]
+
+bb17:
+ %add = add i32 %phi16, 0
+ br label %bb19
+
+bb18:
+ br label %bb19
+
+bb19:
+ %phi20 = phi i32 [ 0, %bb17 ], [ %arg, %bb18 ], [ %phi12, %bb11 ]
+ %phi21 = phi i32 [ %phi13, %bb17 ], [ %phi12, %bb18 ], [ 0, %bb11 ]
+ %phi22 = phi i32 [ %phi14, %bb17 ], [ 0, %bb18 ], [ 0, %bb11 ]
+ %phi23 = phi i32 [ %phi15, %bb17 ], [ %arg, %bb18 ], [ %arg, %bb11 ]
+ %phi24 = phi i32 [ %add, %bb17 ], [ %phi16, %bb18 ], [ %phi16, %bb11 ]
+ br i1 false, label %bb2, label %bb25
+
+bb25:
+ switch i8 0, label %bb6 [
+ i8 0, label %bb26
+ i8 1, label %bb27
+ i8 6, label %bb27
+ ]
+
+bb26:
+ br label %bb6
+
+bb27:
+ br label %bb6
+
+bb28:
+ br label %bb11
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-nodes-incoming-same-blocks.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-nodes-incoming-same-blocks.ll
index d62623047763f..5253f9f36bba5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi-nodes-incoming-same-blocks.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-nodes-incoming-same-blocks.ll
@@ -6,7 +6,7 @@ define void @test(ptr %0, i1 %1, i1 %2) {
; CHECK-SAME: ptr [[TMP0:%.*]], i1 [[TMP1:%.*]], i1 [[TMP2:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: br label %[[BB4:.*]]
; CHECK: [[BB4]]:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ [[TMP12:%.*]], %[[TMP7:.*]] ], [ zeroinitializer, [[TMP3:%.*]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x i32> [ [[TMP15:%.*]], %[[TMP7:.*]] ], [ zeroinitializer, [[TMP3:%.*]] ]
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
; CHECK-NEXT: br i1 [[TMP1]], label %[[TMP7]], label %[[BB15:.*]]
; CHECK: [[TMP7]]:
@@ -14,9 +14,9 @@ define void @test(ptr %0, i1 %1, i1 %2) {
; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 16
; CHECK-NEXT: [[TMP10:%.*]] = load <2 x i32>, ptr [[TMP9]], align 1
; CHECK-NEXT: [[TMP11:%.*]] = or <2 x i32> [[TMP10]], splat (i32 1)
-; CHECK-NEXT: [[TMP12]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> <i32 1, i32 poison>, <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 poison, i32 poison>, <4 x i32> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: [[TMP15]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> <i32 1, i32 poison>, <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: br i1 [[TMP2]], label %[[BB16:.*]], label %[[BB4]]
; CHECK: [[BB15]]:
; CHECK-NEXT: br label %[[BB16]]
More information about the llvm-commits
mailing list