[llvm] [LoopVectorize] Loop-invariant backedge values prevent recognition of fixed-order recurrence (PR #228021)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Oct 1 03:20:07 PDT 2026
https://github.com/yafet-a created https://github.com/llvm/llvm-project/pull/228021
Fixes regression reported in #228018
>From 0c9779a2a8cdf19758f85dbed8681041b8be24bc Mon Sep 17 00:00:00 2001
From: Yafet Beyene <ybeyene at nvidia.com>
Date: Wed, 30 Sep 2026 05:18:16 -0700
Subject: [PATCH 1/2] [LV] Support fixed-order recurrences with invariant
backedge values
---
llvm/lib/Analysis/IVDescriptors.cpp | 6 ++-
...rst-order-recurrence-invariant-backedge.ll | 39 +++++++++++++++++++
2 files changed, 44 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/first-order-recurrence-invariant-backedge.ll
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index d281dd0cfc695..cfc769ddb7222 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -1174,9 +1174,13 @@ bool RecurrenceDescriptor::isFixedOrderRecurrence(PHINode *Phi, Loop *TheLoop,
Previous = dyn_cast<Instruction>(PrevPhi->getIncomingValueForBlock(Latch));
}
- if (!Previous || !TheLoop->contains(Previous) || isa<PHINode>(Previous))
+ if (!Previous || isa<PHINode>(Previous))
return false;
+ // Handle recurrences with a loop-invariant latch value.
+ if (!TheLoop->contains(Previous))
+ return DT->dominates(Previous, Phi);
+
// Ensure every user of the phi node (recursively) is dominated by the
// previous value. The dominance requirement ensures the loop vectorizer will
// not need to vectorize the initial value prior to the first iteration of the
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-invariant-backedge.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-invariant-backedge.ll
new file mode 100644
index 0000000000000..c72c9a5b139e9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-invariant-backedge.ll
@@ -0,0 +1,39 @@
+; RUN: opt -passes=loop-vectorize -force-vector-width=2 \
+; RUN: -force-vector-interleave=1 -S %s | FileCheck %s
+
+; GVN load PRE can create a recurrence whose latch value is loop invariant.
+; Make sure the recurrence is recognized and widened.
+
+; CHECK-LABEL: define void @invariant_backedge_phi(
+; CHECK: vector.ph:
+; CHECK: [[SPLATINSERT:%.*]] = insertelement <2 x double> poison, double %next, i64 0
+; CHECK: [[SPLAT:%.*]] = shufflevector <2 x double> [[SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK: [[RECURINIT:%.*]] = insertelement <2 x double> poison, double %init, i32 1
+; CHECK: vector.body:
+; CHECK: [[RECUR:%.*]] = phi <2 x double> [ [[RECURINIT]], %vector.ph ], [ [[SPLAT]], %vector.body ]
+; CHECK: shufflevector <2 x double> [[RECUR]], <2 x double> [[SPLAT]], <2 x i32> <i32 1, i32 2>
+
+define void @invariant_backedge_phi(
+ ptr noalias %dst, ptr noalias readonly %src,
+ ptr noalias readonly %coeffs, i64 %n) {
+entry:
+ %init = load double, ptr %coeffs, align 8
+ %next.ptr = getelementptr inbounds double, ptr %coeffs, i64 1
+ %next = load double, ptr %next.ptr, align 8
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+ %factor = phi double [ %init, %entry ], [ %next, %loop ]
+ %src.ptr = getelementptr inbounds double, ptr %src, i64 %iv
+ %x = load double, ptr %src.ptr, align 8
+ %result = fmul double %factor, %x
+ %dst.ptr = getelementptr inbounds double, ptr %dst, i64 %iv
+ store double %result, ptr %dst.ptr, align 8
+ %iv.next = add nuw i64 %iv, 1
+ %done = icmp eq i64 %iv.next, %n
+ br i1 %done, label %exit, label %loop
+
+exit:
+ ret void
+}
>From 08d459e6bc6dd2886f1c12bfdd703ea8663eb06a Mon Sep 17 00:00:00 2001
From: Yafet Beyene <ybeyene at nvidia.com>
Date: Wed, 30 Sep 2026 06:01:23 -0700
Subject: [PATCH 2/2] [LV] Update fixed-order recurrence checks
---
.../LoopVectorize/first-order-recurrence.ll | 118 +++++++++++++++---
1 file changed, 103 insertions(+), 15 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
index 17e34d8fa0d33..bb9a5e7cd9367 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence.ll
@@ -725,17 +725,50 @@ define void @PR26734(ptr %a, ptr %b, ptr %c, i32 %d, ptr %e) {
; UNROLL-NO-IC-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32
; UNROLL-NO-IC-NEXT: [[C_PROMOTED:%.*]] = load i32, ptr [[C]], align 4
; UNROLL-NO-IC-NEXT: [[B_PROMOTED:%.*]] = load i32, ptr [[B]], align 4
+; UNROLL-NO-IC-NEXT: [[TMP1:%.*]] = sub i32 21, [[D]]
+; UNROLL-NO-IC-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 8
+; UNROLL-NO-IC-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL-NO-IC: [[VECTOR_PH]]:
+; UNROLL-NO-IC-NEXT: [[TMP2:%.*]] = and i32 [[TMP1]], 7
+; UNROLL-NO-IC-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP2]]
+; UNROLL-NO-IC-NEXT: [[TMP3:%.*]] = add i32 [[D]], [[N_VEC]]
+; UNROLL-NO-IC-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> splat (i32 -1), i32 [[B_PROMOTED]], i64 0
+; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[CONV2]], i64 0
+; UNROLL-NO-IC-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[C_PROMOTED]], i32 3
; UNROLL-NO-IC-NEXT: br label %[[FOR_BODY:.*]]
; UNROLL-NO-IC: [[FOR_BODY]]:
-; UNROLL-NO-IC-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], %[[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; UNROLL-NO-IC-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], %[[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], %[[FOR_BODY]] ]
-; UNROLL-NO-IC-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], %[[FOR_BODY_LR_PH]] ], [ [[CONV2]], %[[FOR_BODY]] ]
+; UNROLL-NO-IC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; UNROLL-NO-IC-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[TMP7:%.*]], %[[FOR_BODY]] ]
+; UNROLL-NO-IC-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[FOR_BODY]] ]
+; UNROLL-NO-IC-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT]], %[[FOR_BODY]] ]
+; UNROLL-NO-IC-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; UNROLL-NO-IC-NEXT: [[TMP7]] = and <4 x i32> [[VEC_PHI]], [[TMP5]]
+; UNROLL-NO-IC-NEXT: [[TMP8]] = and <4 x i32> [[VEC_PHI1]], [[TMP6]]
+; UNROLL-NO-IC-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; UNROLL-NO-IC-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; UNROLL-NO-IC-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; UNROLL-NO-IC: [[MIDDLE_BLOCK]]:
+; UNROLL-NO-IC-NEXT: [[BIN_RDX:%.*]] = and <4 x i32> [[TMP8]], [[TMP7]]
+; UNROLL-NO-IC-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[BIN_RDX]])
+; UNROLL-NO-IC-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; UNROLL-NO-IC-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_FOR_END_CRIT_EDGE:.*]], label %[[SCALAR_PH]]
+; UNROLL-NO-IC: [[SCALAR_PH]]:
+; UNROLL-NO-IC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[D]], %[[FOR_BODY_LR_PH]] ]
+; UNROLL-NO-IC-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[B_PROMOTED]], %[[FOR_BODY_LR_PH]] ]
+; UNROLL-NO-IC-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[CONV2]], %[[MIDDLE_BLOCK]] ], [ [[C_PROMOTED]], %[[FOR_BODY_LR_PH]] ]
+; UNROLL-NO-IC-NEXT: br label %[[FOR_BODY1:.*]]
+; UNROLL-NO-IC: [[FOR_BODY1]]:
+; UNROLL-NO-IC-NEXT: [[INC7:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY1]] ]
+; UNROLL-NO-IC-NEXT: [[AND6:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[AND:%.*]], %[[FOR_BODY1]] ]
+; UNROLL-NO-IC-NEXT: [[CONV25:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[CONV2]], %[[FOR_BODY1]] ]
; UNROLL-NO-IC-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]]
; UNROLL-NO-IC-NEXT: [[INC]] = add nsw i32 [[INC7]], 1
; UNROLL-NO-IC-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21
-; UNROLL-NO-IC-NEXT: br i1 [[CMP]], label %[[FOR_COND_FOR_END_CRIT_EDGE:.*]], label %[[FOR_BODY]]
+; UNROLL-NO-IC-NEXT: br i1 [[CMP]], label %[[FOR_COND_FOR_END_CRIT_EDGE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
; UNROLL-NO-IC: [[FOR_COND_FOR_END_CRIT_EDGE]]:
-; UNROLL-NO-IC-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], %[[FOR_BODY]] ]
+; UNROLL-NO-IC-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], %[[FOR_BODY1]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
; UNROLL-NO-IC-NEXT: store i32 [[CONV2]], ptr [[C]], align 4
; UNROLL-NO-IC-NEXT: store i32 [[AND_LCSSA]], ptr [[B]], align 4
; UNROLL-NO-IC-NEXT: store i16 [[SUB]], ptr [[E]], align 2
@@ -757,17 +790,43 @@ define void @PR26734(ptr %a, ptr %b, ptr %c, i32 %d, ptr %e) {
; UNROLL-NO-VF-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32
; UNROLL-NO-VF-NEXT: [[C_PROMOTED:%.*]] = load i32, ptr [[C]], align 4
; UNROLL-NO-VF-NEXT: [[B_PROMOTED:%.*]] = load i32, ptr [[B]], align 4
+; UNROLL-NO-VF-NEXT: [[TMP1:%.*]] = sub i32 21, [[D]]
+; UNROLL-NO-VF-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 2
+; UNROLL-NO-VF-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; UNROLL-NO-VF: [[VECTOR_PH]]:
+; UNROLL-NO-VF-NEXT: [[TMP2:%.*]] = and i32 [[TMP1]], 1
+; UNROLL-NO-VF-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP2]]
+; UNROLL-NO-VF-NEXT: [[TMP3:%.*]] = add i32 [[D]], [[N_VEC]]
; UNROLL-NO-VF-NEXT: br label %[[FOR_BODY:.*]]
; UNROLL-NO-VF: [[FOR_BODY]]:
-; UNROLL-NO-VF-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], %[[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; UNROLL-NO-VF-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], %[[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], %[[FOR_BODY]] ]
-; UNROLL-NO-VF-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], %[[FOR_BODY_LR_PH]] ], [ [[CONV2]], %[[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], %[[VECTOR_PH]] ], [ [[AND:%.*]], %[[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ -1, %[[VECTOR_PH]] ], [ [[TMP5:%.*]], %[[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], %[[VECTOR_PH]] ], [ [[CONV2]], %[[FOR_BODY]] ]
; UNROLL-NO-VF-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]]
+; UNROLL-NO-VF-NEXT: [[TMP5]] = and i32 [[VEC_PHI1]], [[CONV2]]
+; UNROLL-NO-VF-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2
+; UNROLL-NO-VF-NEXT: [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; UNROLL-NO-VF-NEXT: br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; UNROLL-NO-VF: [[MIDDLE_BLOCK]]:
+; UNROLL-NO-VF-NEXT: [[BIN_RDX:%.*]] = and i32 [[TMP5]], [[AND]]
+; UNROLL-NO-VF-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; UNROLL-NO-VF-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_FOR_END_CRIT_EDGE:.*]], label %[[SCALAR_PH]]
+; UNROLL-NO-VF: [[SCALAR_PH]]:
+; UNROLL-NO-VF-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[D]], %[[FOR_BODY_LR_PH]] ]
+; UNROLL-NO-VF-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ], [ [[B_PROMOTED]], %[[FOR_BODY_LR_PH]] ]
+; UNROLL-NO-VF-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[CONV2]], %[[MIDDLE_BLOCK]] ], [ [[C_PROMOTED]], %[[FOR_BODY_LR_PH]] ]
+; UNROLL-NO-VF-NEXT: br label %[[FOR_BODY1:.*]]
+; UNROLL-NO-VF: [[FOR_BODY1]]:
+; UNROLL-NO-VF-NEXT: [[INC7:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY1]] ]
+; UNROLL-NO-VF-NEXT: [[AND7:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[AND1:%.*]], %[[FOR_BODY1]] ]
+; UNROLL-NO-VF-NEXT: [[CONV26:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[CONV2]], %[[FOR_BODY1]] ]
+; UNROLL-NO-VF-NEXT: [[AND1]] = and i32 [[AND7]], [[CONV26]]
; UNROLL-NO-VF-NEXT: [[INC]] = add nsw i32 [[INC7]], 1
; UNROLL-NO-VF-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21
-; UNROLL-NO-VF-NEXT: br i1 [[CMP]], label %[[FOR_COND_FOR_END_CRIT_EDGE:.*]], label %[[FOR_BODY]]
+; UNROLL-NO-VF-NEXT: br i1 [[CMP]], label %[[FOR_COND_FOR_END_CRIT_EDGE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
; UNROLL-NO-VF: [[FOR_COND_FOR_END_CRIT_EDGE]]:
-; UNROLL-NO-VF-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], %[[FOR_BODY]] ]
+; UNROLL-NO-VF-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND1]], %[[FOR_BODY1]] ], [ [[BIN_RDX]], %[[MIDDLE_BLOCK]] ]
; UNROLL-NO-VF-NEXT: store i32 [[CONV2]], ptr [[C]], align 4
; UNROLL-NO-VF-NEXT: store i32 [[AND_LCSSA]], ptr [[B]], align 4
; UNROLL-NO-VF-NEXT: store i16 [[SUB]], ptr [[E]], align 2
@@ -789,17 +848,46 @@ define void @PR26734(ptr %a, ptr %b, ptr %c, i32 %d, ptr %e) {
; SINK-AFTER-NEXT: [[CONV2:%.*]] = sext i16 [[SUB]] to i32
; SINK-AFTER-NEXT: [[C_PROMOTED:%.*]] = load i32, ptr [[C]], align 4
; SINK-AFTER-NEXT: [[B_PROMOTED:%.*]] = load i32, ptr [[B]], align 4
+; SINK-AFTER-NEXT: [[TMP1:%.*]] = sub i32 21, [[D]]
+; SINK-AFTER-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], 4
+; SINK-AFTER-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; SINK-AFTER: [[VECTOR_PH]]:
+; SINK-AFTER-NEXT: [[TMP2:%.*]] = and i32 [[TMP1]], 3
+; SINK-AFTER-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[TMP2]]
+; SINK-AFTER-NEXT: [[TMP3:%.*]] = add i32 [[D]], [[N_VEC]]
+; SINK-AFTER-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> splat (i32 -1), i32 [[B_PROMOTED]], i64 0
+; SINK-AFTER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[CONV2]], i64 0
+; SINK-AFTER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
+; SINK-AFTER-NEXT: [[VECTOR_RECUR_INIT:%.*]] = insertelement <4 x i32> poison, i32 [[C_PROMOTED]], i32 3
; SINK-AFTER-NEXT: br label %[[FOR_BODY:.*]]
; SINK-AFTER: [[FOR_BODY]]:
-; SINK-AFTER-NEXT: [[INC7:%.*]] = phi i32 [ [[D]], %[[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ]
-; SINK-AFTER-NEXT: [[AND6:%.*]] = phi i32 [ [[B_PROMOTED]], %[[FOR_BODY_LR_PH]] ], [ [[AND:%.*]], %[[FOR_BODY]] ]
-; SINK-AFTER-NEXT: [[CONV25:%.*]] = phi i32 [ [[C_PROMOTED]], %[[FOR_BODY_LR_PH]] ], [ [[CONV2]], %[[FOR_BODY]] ]
+; SINK-AFTER-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[FOR_BODY]] ]
+; SINK-AFTER-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[TMP6:%.*]], %[[FOR_BODY]] ]
+; SINK-AFTER-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[BROADCAST_SPLAT]], %[[FOR_BODY]] ]
+; SINK-AFTER-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[BROADCAST_SPLAT]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; SINK-AFTER-NEXT: [[TMP6]] = and <4 x i32> [[VEC_PHI]], [[TMP5]]
+; SINK-AFTER-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; SINK-AFTER-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; SINK-AFTER-NEXT: br i1 [[TMP7]], label %[[MIDDLE_BLOCK:.*]], label %[[FOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; SINK-AFTER: [[MIDDLE_BLOCK]]:
+; SINK-AFTER-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP6]])
+; SINK-AFTER-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
+; SINK-AFTER-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_FOR_END_CRIT_EDGE:.*]], label %[[SCALAR_PH]]
+; SINK-AFTER: [[SCALAR_PH]]:
+; SINK-AFTER-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[D]], %[[FOR_BODY_LR_PH]] ]
+; SINK-AFTER-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ [[B_PROMOTED]], %[[FOR_BODY_LR_PH]] ]
+; SINK-AFTER-NEXT: [[SCALAR_RECUR_INIT:%.*]] = phi i32 [ [[CONV2]], %[[MIDDLE_BLOCK]] ], [ [[C_PROMOTED]], %[[FOR_BODY_LR_PH]] ]
+; SINK-AFTER-NEXT: br label %[[FOR_BODY1:.*]]
+; SINK-AFTER: [[FOR_BODY1]]:
+; SINK-AFTER-NEXT: [[INC7:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY1]] ]
+; SINK-AFTER-NEXT: [[AND6:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[AND:%.*]], %[[FOR_BODY1]] ]
+; SINK-AFTER-NEXT: [[CONV25:%.*]] = phi i32 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[CONV2]], %[[FOR_BODY1]] ]
; SINK-AFTER-NEXT: [[AND]] = and i32 [[AND6]], [[CONV25]]
; SINK-AFTER-NEXT: [[INC]] = add nsw i32 [[INC7]], 1
; SINK-AFTER-NEXT: [[CMP:%.*]] = icmp eq i32 [[INC]], 21
-; SINK-AFTER-NEXT: br i1 [[CMP]], label %[[FOR_COND_FOR_END_CRIT_EDGE:.*]], label %[[FOR_BODY]]
+; SINK-AFTER-NEXT: br i1 [[CMP]], label %[[FOR_COND_FOR_END_CRIT_EDGE]], label %[[FOR_BODY1]], !llvm.loop [[LOOP9:![0-9]+]]
; SINK-AFTER: [[FOR_COND_FOR_END_CRIT_EDGE]]:
-; SINK-AFTER-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], %[[FOR_BODY]] ]
+; SINK-AFTER-NEXT: [[AND_LCSSA:%.*]] = phi i32 [ [[AND]], %[[FOR_BODY1]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ]
; SINK-AFTER-NEXT: store i32 [[CONV2]], ptr [[C]], align 4
; SINK-AFTER-NEXT: store i32 [[AND_LCSSA]], ptr [[B]], align 4
; SINK-AFTER-NEXT: store i16 [[SUB]], ptr [[E]], align 2
More information about the llvm-commits
mailing list