[llvm] [LV] Simplify live-out extraction for first-order recurrence phis when tail folding (PR #176108)
Mel Chen via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 15 02:58:03 PDT 2026
https://github.com/Mel-Chen updated https://github.com/llvm/llvm-project/pull/176108
>From 4b015362f91cf03a275304c06258322db46fc658 Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen at sifive.com>
Date: Mon, 12 Jan 2026 19:26:30 -0800
Subject: [PATCH] [LV] Simplify first-order recurrence live-out extract when
tail folding
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 26 --------
.../first-order-recurrence-scalable-vf1.ll | 9 +--
.../tail-folding-fixed-order-recurrence.ll | 9 +--
.../first-order-recurrence-tail-folding.ll | 60 ++++++-------------
llvm/test/Transforms/LoopVectorize/optsize.ll | 45 +++++---------
.../tail-folding-vectorization-factor-1.ll | 20 +++----
6 files changed, 46 insertions(+), 123 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 738f6bc92f541..6e6cc9c700134 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2512,32 +2512,6 @@ bool VPlanTransforms::adjustFixedOrderRecurrences(VPlan &Plan,
// Set the first operand of RecurSplice to FOR again, after replacing
// all users.
RecurSplice->setOperand(0, FOR);
-
- // Check for users extracting at the penultimate active lane of the FOR.
- // If only a single lane is active in the current iteration, we need to
- // select the last element from the previous iteration (from the FOR phi
- // directly).
- for (VPUser *U : RecurSplice->users()) {
- if (!match(U, m_ExtractLane(m_LastActiveLane(m_VPValue()),
- m_Specific(RecurSplice))))
- continue;
-
- VPBuilder B(cast<VPInstruction>(U));
- VPValue *LastActiveLane = cast<VPInstruction>(U)->getOperand(0);
- Type *Ty = TypeInfo.inferScalarType(LastActiveLane);
- VPValue *Zero = Plan.getConstantInt(Ty, 0);
- VPValue *One = Plan.getConstantInt(Ty, 1);
- VPValue *PenultimateIndex = B.createSub(LastActiveLane, One);
- VPValue *PenultimateLastIter =
- B.createNaryOp(VPInstruction::ExtractLane,
- {PenultimateIndex, FOR->getBackedgeValue()});
- VPValue *LastPrevIter =
- B.createNaryOp(VPInstruction::ExtractLastLane, FOR);
-
- VPValue *Cmp = B.createICmp(CmpInst::ICMP_EQ, LastActiveLane, Zero);
- VPValue *Sel = B.createSelect(Cmp, LastPrevIter, PenultimateLastIter);
- cast<VPInstruction>(U)->replaceAllUsesWith(Sel);
- }
}
return true;
}
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll b/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
index d28d33bb4eaec..5fe69bb581e47 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
@@ -36,14 +36,7 @@ define i64 @pr97452_scalable_vf1_for(ptr %src, ptr noalias %dst) #0 {
; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP15:%.*]] = sub i64 [[TMP12]], 1
-; CHECK-NEXT: [[TMP16:%.*]] = sub i64 [[TMP15]], 1
-; CHECK-NEXT: [[TMP20:%.*]] = extractelement <vscale x 2 x i64> [[VP_OP_LOAD]], i64 [[TMP16]]
-; CHECK-NEXT: [[TMP21:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: [[TMP22:%.*]] = mul nuw i32 [[TMP21]], 2
-; CHECK-NEXT: [[TMP23:%.*]] = sub i32 [[TMP22]], 1
-; CHECK-NEXT: [[TMP24:%.*]] = extractelement <vscale x 2 x i64> [[VECTOR_RECUR]], i32 [[TMP23]]
-; CHECK-NEXT: [[TMP25:%.*]] = icmp eq i64 [[TMP15]], 0
-; CHECK-NEXT: [[TMP26:%.*]] = select i1 [[TMP25]], i64 [[TMP24]], i64 [[TMP20]]
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <vscale x 2 x i64> [[TMP10]], i64 [[TMP15]]
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i64 [[TMP26]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll
index 6f7aca9767bc1..cef18b9985786 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-fixed-order-recurrence.ll
@@ -430,14 +430,7 @@ define i32 @FOR_reduction(ptr noalias %A, ptr noalias %B, i64 %TC) {
; IF-EVL-NEXT: br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; IF-EVL: [[MIDDLE_BLOCK]]:
; IF-EVL-NEXT: [[TMP28:%.*]] = sub i64 [[TMP13]], 1
-; IF-EVL-NEXT: [[TMP17:%.*]] = sub i64 [[TMP28]], 1
-; IF-EVL-NEXT: [[TMP21:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i64 [[TMP17]]
-; IF-EVL-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT: [[TMP15:%.*]] = mul nuw i32 [[TMP14]], 4
-; IF-EVL-NEXT: [[TMP16:%.*]] = sub i32 [[TMP15]], 1
-; IF-EVL-NEXT: [[TMP25:%.*]] = extractelement <vscale x 4 x i32> [[VECTOR_RECUR]], i32 [[TMP16]]
-; IF-EVL-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP28]], 0
-; IF-EVL-NEXT: [[FOR1_LCSSA:%.*]] = select i1 [[TMP26]], i32 [[TMP25]], i32 [[TMP21]]
+; IF-EVL-NEXT: [[FOR1_LCSSA:%.*]] = extractelement <vscale x 4 x i32> [[TMP10]], i64 [[TMP28]]
; IF-EVL-NEXT: br label %[[FOR_END:.*]]
; IF-EVL: [[FOR_END]]:
; IF-EVL-NEXT: ret i32 [[FOR1_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
index c8f59f5f2b6f9..c950a16f6de83 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-tail-folding.ll
@@ -36,11 +36,7 @@ define i32 @FOR_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF2IC1-NEXT: [[TMP25:%.*]] = xor <2 x i1> [[TMP2]], splat (i1 true)
; VF2IC1-NEXT: [[TMP26:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP25]], i1 false)
; VF2IC1-NEXT: [[TMP27:%.*]] = sub i64 [[TMP26]], 1
-; VF2IC1-NEXT: [[TMP28:%.*]] = sub i64 [[TMP27]], 1
-; VF2IC1-NEXT: [[TMP29:%.*]] = extractelement <2 x i32> [[TMP12]], i64 [[TMP28]]
-; VF2IC1-NEXT: [[TMP30:%.*]] = extractelement <2 x i32> [[VECTOR_RECUR]], i32 1
-; VF2IC1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP27]], 0
-; VF2IC1-NEXT: [[TMP32:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 [[TMP29]]
+; VF2IC1-NEXT: [[TMP32:%.*]] = extractelement <2 x i32> [[TMP13]], i64 [[TMP27]]
; VF2IC1-NEXT: br label %[[FOR_END:.*]]
; VF2IC1: [[FOR_END]]:
; VF2IC1-NEXT: ret i32 [[TMP32]]
@@ -90,16 +86,12 @@ define i32 @FOR_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF2IC2-NEXT: [[TMP54:%.*]] = add i64 0, [[TMP53]]
; VF2IC2-NEXT: [[TMP55:%.*]] = icmp ne i64 [[TMP53]], 2
; VF2IC2-NEXT: [[TMP56:%.*]] = select i1 [[TMP55]], i64 [[TMP54]], i64 [[TMP52]]
-; VF2IC2-NEXT: [[TMP57:%.*]] = sub i64 [[TMP56]], 1
-; VF2IC2-NEXT: [[TMP58:%.*]] = sub i64 [[TMP57]], 1
-; VF2IC2-NEXT: [[TMP59:%.*]] = extractelement <2 x i32> [[TMP15]], i64 [[TMP58]]
+; VF2IC2-NEXT: [[TMP58:%.*]] = sub i64 [[TMP56]], 1
+; VF2IC2-NEXT: [[TMP59:%.*]] = extractelement <2 x i32> [[TMP26]], i64 [[TMP58]]
; VF2IC2-NEXT: [[TMP60:%.*]] = sub i64 [[TMP58]], 2
-; VF2IC2-NEXT: [[TMP61:%.*]] = extractelement <2 x i32> [[TMP25]], i64 [[TMP60]]
+; VF2IC2-NEXT: [[TMP57:%.*]] = extractelement <2 x i32> [[TMP27]], i64 [[TMP60]]
; VF2IC2-NEXT: [[TMP62:%.*]] = icmp uge i64 [[TMP58]], 2
-; VF2IC2-NEXT: [[TMP63:%.*]] = select i1 [[TMP62]], i32 [[TMP61]], i32 [[TMP59]]
-; VF2IC2-NEXT: [[TMP64:%.*]] = extractelement <2 x i32> [[VECTOR_RECUR]], i32 1
-; VF2IC2-NEXT: [[TMP65:%.*]] = icmp eq i64 [[TMP57]], 0
-; VF2IC2-NEXT: [[TMP66:%.*]] = select i1 [[TMP65]], i32 [[TMP64]], i32 [[TMP63]]
+; VF2IC2-NEXT: [[TMP66:%.*]] = select i1 [[TMP62]], i32 [[TMP57]], i32 [[TMP59]]
; VF2IC2-NEXT: br label %[[FOR_END:.*]]
; VF2IC2: [[FOR_END]]:
; VF2IC2-NEXT: ret i32 [[TMP66]]
@@ -163,13 +155,10 @@ define i32 @FOR_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2-NEXT: [[TMP21:%.*]] = add i64 0, [[TMP20]]
; VF1IC2-NEXT: [[TMP22:%.*]] = icmp ne i64 [[TMP20]], 1
; VF1IC2-NEXT: [[TMP23:%.*]] = select i1 [[TMP22]], i64 [[TMP21]], i64 [[TMP18]]
-; VF1IC2-NEXT: [[TMP24:%.*]] = sub i64 [[TMP23]], 1
-; VF1IC2-NEXT: [[TMP25:%.*]] = sub i64 [[TMP24]], 1
+; VF1IC2-NEXT: [[TMP25:%.*]] = sub i64 [[TMP23]], 1
; VF1IC2-NEXT: [[TMP26:%.*]] = sub i64 [[TMP25]], 1
; VF1IC2-NEXT: [[TMP27:%.*]] = icmp uge i64 [[TMP25]], 1
-; VF1IC2-NEXT: [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP8]], i32 [[TMP5]]
-; VF1IC2-NEXT: [[TMP29:%.*]] = icmp eq i64 [[TMP24]], 0
-; VF1IC2-NEXT: [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[VECTOR_RECUR]], i32 [[TMP28]]
+; VF1IC2-NEXT: [[TMP30:%.*]] = select i1 [[TMP27]], i32 [[TMP5]], i32 [[VECTOR_RECUR]]
; VF1IC2-NEXT: br label %[[FOR_END:.*]]
; VF1IC2: [[FOR_END]]:
; VF1IC2-NEXT: ret i32 [[TMP30]]
@@ -405,11 +394,7 @@ define i32 @FOR_and_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF2IC1-NEXT: [[TMP25:%.*]] = xor <2 x i1> [[TMP2]], splat (i1 true)
; VF2IC1-NEXT: [[TMP26:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v2i1(<2 x i1> [[TMP25]], i1 false)
; VF2IC1-NEXT: [[TMP27:%.*]] = sub i64 [[TMP26]], 1
-; VF2IC1-NEXT: [[TMP28:%.*]] = sub i64 [[TMP27]], 1
-; VF2IC1-NEXT: [[TMP29:%.*]] = extractelement <2 x i32> [[TMP12]], i64 [[TMP28]]
-; VF2IC1-NEXT: [[TMP30:%.*]] = extractelement <2 x i32> [[VECTOR_RECUR]], i32 1
-; VF2IC1-NEXT: [[TMP31:%.*]] = icmp eq i64 [[TMP27]], 0
-; VF2IC1-NEXT: [[TMP32:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 [[TMP29]]
+; VF2IC1-NEXT: [[TMP32:%.*]] = extractelement <2 x i32> [[TMP13]], i64 [[TMP27]]
; VF2IC1-NEXT: [[TMP33:%.*]] = extractelement <2 x i32> [[TMP12]], i64 [[TMP27]]
; VF2IC1-NEXT: br label %[[FOR_END:.*]]
; VF2IC1: [[FOR_END]]:
@@ -461,20 +446,16 @@ define i32 @FOR_and_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF2IC2-NEXT: [[TMP54:%.*]] = add i64 0, [[TMP53]]
; VF2IC2-NEXT: [[TMP55:%.*]] = icmp ne i64 [[TMP53]], 2
; VF2IC2-NEXT: [[TMP56:%.*]] = select i1 [[TMP55]], i64 [[TMP54]], i64 [[TMP52]]
-; VF2IC2-NEXT: [[TMP57:%.*]] = sub i64 [[TMP56]], 1
-; VF2IC2-NEXT: [[TMP58:%.*]] = sub i64 [[TMP57]], 1
-; VF2IC2-NEXT: [[TMP59:%.*]] = extractelement <2 x i32> [[TMP15]], i64 [[TMP58]]
+; VF2IC2-NEXT: [[TMP58:%.*]] = sub i64 [[TMP56]], 1
+; VF2IC2-NEXT: [[TMP59:%.*]] = extractelement <2 x i32> [[TMP26]], i64 [[TMP58]]
; VF2IC2-NEXT: [[TMP60:%.*]] = sub i64 [[TMP58]], 2
-; VF2IC2-NEXT: [[TMP61:%.*]] = extractelement <2 x i32> [[TMP25]], i64 [[TMP60]]
+; VF2IC2-NEXT: [[TMP57:%.*]] = extractelement <2 x i32> [[TMP27]], i64 [[TMP60]]
; VF2IC2-NEXT: [[TMP62:%.*]] = icmp uge i64 [[TMP58]], 2
-; VF2IC2-NEXT: [[TMP63:%.*]] = select i1 [[TMP62]], i32 [[TMP61]], i32 [[TMP59]]
-; VF2IC2-NEXT: [[TMP64:%.*]] = extractelement <2 x i32> [[VECTOR_RECUR]], i32 1
-; VF2IC2-NEXT: [[TMP65:%.*]] = icmp eq i64 [[TMP57]], 0
-; VF2IC2-NEXT: [[TMP66:%.*]] = select i1 [[TMP65]], i32 [[TMP64]], i32 [[TMP63]]
-; VF2IC2-NEXT: [[TMP67:%.*]] = extractelement <2 x i32> [[TMP15]], i64 [[TMP57]]
-; VF2IC2-NEXT: [[TMP68:%.*]] = sub i64 [[TMP57]], 2
+; VF2IC2-NEXT: [[TMP66:%.*]] = select i1 [[TMP62]], i32 [[TMP57]], i32 [[TMP59]]
+; VF2IC2-NEXT: [[TMP67:%.*]] = extractelement <2 x i32> [[TMP15]], i64 [[TMP58]]
+; VF2IC2-NEXT: [[TMP68:%.*]] = sub i64 [[TMP58]], 2
; VF2IC2-NEXT: [[TMP69:%.*]] = extractelement <2 x i32> [[TMP25]], i64 [[TMP68]]
-; VF2IC2-NEXT: [[TMP70:%.*]] = icmp uge i64 [[TMP57]], 2
+; VF2IC2-NEXT: [[TMP70:%.*]] = icmp uge i64 [[TMP58]], 2
; VF2IC2-NEXT: [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[TMP69]], i32 [[TMP67]]
; VF2IC2-NEXT: br label %[[FOR_END:.*]]
; VF2IC2: [[FOR_END]]:
@@ -540,15 +521,12 @@ define i32 @FOR_and_next_used_outside(ptr noalias %A, ptr noalias %B, i64 %n) {
; VF1IC2-NEXT: [[TMP21:%.*]] = add i64 0, [[TMP20]]
; VF1IC2-NEXT: [[TMP22:%.*]] = icmp ne i64 [[TMP20]], 1
; VF1IC2-NEXT: [[TMP23:%.*]] = select i1 [[TMP22]], i64 [[TMP21]], i64 [[TMP18]]
-; VF1IC2-NEXT: [[TMP24:%.*]] = sub i64 [[TMP23]], 1
-; VF1IC2-NEXT: [[TMP25:%.*]] = sub i64 [[TMP24]], 1
+; VF1IC2-NEXT: [[TMP25:%.*]] = sub i64 [[TMP23]], 1
; VF1IC2-NEXT: [[TMP26:%.*]] = sub i64 [[TMP25]], 1
; VF1IC2-NEXT: [[TMP27:%.*]] = icmp uge i64 [[TMP25]], 1
-; VF1IC2-NEXT: [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP8]], i32 [[TMP5]]
-; VF1IC2-NEXT: [[TMP29:%.*]] = icmp eq i64 [[TMP24]], 0
-; VF1IC2-NEXT: [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[VECTOR_RECUR]], i32 [[TMP28]]
-; VF1IC2-NEXT: [[TMP31:%.*]] = sub i64 [[TMP24]], 1
-; VF1IC2-NEXT: [[TMP32:%.*]] = icmp uge i64 [[TMP24]], 1
+; VF1IC2-NEXT: [[TMP30:%.*]] = select i1 [[TMP27]], i32 [[TMP5]], i32 [[VECTOR_RECUR]]
+; VF1IC2-NEXT: [[TMP28:%.*]] = sub i64 [[TMP25]], 1
+; VF1IC2-NEXT: [[TMP32:%.*]] = icmp uge i64 [[TMP25]], 1
; VF1IC2-NEXT: [[TMP33:%.*]] = select i1 [[TMP32]], i32 [[TMP8]], i32 [[TMP5]]
; VF1IC2-NEXT: br label %[[FOR_END:.*]]
; VF1IC2: [[FOR_END]]:
diff --git a/llvm/test/Transforms/LoopVectorize/optsize.ll b/llvm/test/Transforms/LoopVectorize/optsize.ll
index 476dc466fa10e..cd72bc7e3c3a6 100644
--- a/llvm/test/Transforms/LoopVectorize/optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/optsize.ll
@@ -483,7 +483,7 @@ define i32 @pr45526() optsize {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
@@ -491,13 +491,10 @@ define i32 @pr45526() optsize {
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> [[TMP0]], i1 false)
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i32 [[FIRST_INACTIVE_LANE]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[LAST_ACTIVE_LANE]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[LAST_ACTIVE_LANE]], 0
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i32 [[LAST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 [[TMP10]]
@@ -511,7 +508,7 @@ define i32 @pr45526() optsize {
; PGSO: [[VECTOR_BODY]]:
; PGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PGSO-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; PGSO-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; PGSO-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; PGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; PGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
@@ -519,13 +516,10 @@ define i32 @pr45526() optsize {
; PGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]]
; PGSO: [[MIDDLE_BLOCK]]:
; PGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
+; PGSO-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; PGSO-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> [[TMP0]], i1 false)
; PGSO-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i32 [[FIRST_INACTIVE_LANE]], 1
-; PGSO-NEXT: [[TMP4:%.*]] = sub i32 [[LAST_ACTIVE_LANE]], 1
-; PGSO-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 [[TMP4]]
-; PGSO-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
-; PGSO-NEXT: [[TMP9:%.*]] = icmp eq i32 [[LAST_ACTIVE_LANE]], 0
-; PGSO-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]
+; PGSO-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i32 [[LAST_ACTIVE_LANE]]
; PGSO-NEXT: br label %[[EXIT:.*]]
; PGSO: [[EXIT]]:
; PGSO-NEXT: ret i32 [[TMP10]]
@@ -539,7 +533,7 @@ define i32 @pr45526() optsize {
; NPGSO: [[VECTOR_BODY]]:
; NPGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; NPGSO-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; NPGSO-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; NPGSO-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; NPGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; NPGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; NPGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
@@ -547,13 +541,10 @@ define i32 @pr45526() optsize {
; NPGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
; NPGSO: [[MIDDLE_BLOCK]]:
; NPGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
+; NPGSO-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; NPGSO-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> [[TMP0]], i1 false)
; NPGSO-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i32 [[FIRST_INACTIVE_LANE]], 1
-; NPGSO-NEXT: [[TMP4:%.*]] = sub i32 [[LAST_ACTIVE_LANE]], 1
-; NPGSO-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 [[TMP4]]
-; NPGSO-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
-; NPGSO-NEXT: [[TMP9:%.*]] = icmp eq i32 [[LAST_ACTIVE_LANE]], 0
-; NPGSO-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]
+; NPGSO-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i32 [[LAST_ACTIVE_LANE]]
; NPGSO-NEXT: br label %[[EXIT:.*]]
; NPGSO: [[EXIT]]:
; NPGSO-NEXT: ret i32 [[TMP10]]
@@ -583,7 +574,7 @@ define i32 @pr45526_pgso() !prof !14 {
; CHECK: [[VECTOR_BODY]]:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
@@ -591,13 +582,10 @@ define i32 @pr45526_pgso() !prof !14 {
; CHECK-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> [[TMP0]], i1 false)
; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i32 [[FIRST_INACTIVE_LANE]], 1
-; CHECK-NEXT: [[TMP4:%.*]] = sub i32 [[LAST_ACTIVE_LANE]], 1
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 [[TMP4]]
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
-; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[LAST_ACTIVE_LANE]], 0
-; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i32 [[LAST_ACTIVE_LANE]]
; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
; CHECK-NEXT: ret i32 [[TMP10]]
@@ -611,7 +599,7 @@ define i32 @pr45526_pgso() !prof !14 {
; PGSO: [[VECTOR_BODY]]:
; PGSO-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; PGSO-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; PGSO-NEXT: [[TMP2:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
+; PGSO-NEXT: [[VECTOR_RECUR:%.*]] = phi <4 x i32> [ <i32 poison, i32 poison, i32 poison, i32 5>, %[[VECTOR_PH]] ], [ [[TMP1:%.*]], %[[VECTOR_BODY]] ]
; PGSO-NEXT: [[TMP1]] = add nuw nsw <4 x i32> [[VEC_IND]], splat (i32 1)
; PGSO-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; PGSO-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
@@ -619,13 +607,10 @@ define i32 @pr45526_pgso() !prof !14 {
; PGSO-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
; PGSO: [[MIDDLE_BLOCK]]:
; PGSO-NEXT: [[TMP0:%.*]] = icmp ugt <4 x i32> [[VEC_IND]], splat (i32 510)
+; PGSO-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[VECTOR_RECUR]], <4 x i32> [[TMP1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; PGSO-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i32 @llvm.experimental.cttz.elts.i32.v4i1(<4 x i1> [[TMP0]], i1 false)
; PGSO-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i32 [[FIRST_INACTIVE_LANE]], 1
-; PGSO-NEXT: [[TMP4:%.*]] = sub i32 [[LAST_ACTIVE_LANE]], 1
-; PGSO-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 [[TMP4]]
-; PGSO-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
-; PGSO-NEXT: [[TMP9:%.*]] = icmp eq i32 [[LAST_ACTIVE_LANE]], 0
-; PGSO-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]
+; PGSO-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[TMP4]], i32 [[LAST_ACTIVE_LANE]]
; PGSO-NEXT: br label %[[EXIT:.*]]
; PGSO: [[EXIT]]:
; PGSO-NEXT: ret i32 [[TMP10]]
diff --git a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
index 57e0e9b63b8de..7da7000cea865 100644
--- a/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
+++ b/llvm/test/Transforms/LoopVectorize/tail-folding-vectorization-factor-1.ll
@@ -166,6 +166,10 @@ define i64 @live_out_scalar_vf(i64 %n) {
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK: middle.block:
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[VEC_IND]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <4 x i64> [[VEC_IND]], <4 x i64> [[STEP_ADD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <4 x i64> [[STEP_ADD]], <4 x i64> [[STEP_ADD_2]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <4 x i64> [[STEP_ADD_2]], <4 x i64> [[STEP_ADD_3]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP2:%.*]] = icmp ugt <4 x i64> [[STEP_ADD]], [[BROADCAST_SPLAT]]
; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <4 x i64> [[STEP_ADD_2]], [[BROADCAST_SPLAT]]
@@ -184,24 +188,20 @@ define i64 @live_out_scalar_vf(i64 %n) {
; CHECK-NEXT: [[TMP13:%.*]] = add i64 0, [[FIRST_INACTIVE_LANE3]]
; CHECK-NEXT: [[TMP14:%.*]] = icmp ne i64 [[FIRST_INACTIVE_LANE3]], 4
; CHECK-NEXT: [[TMP15:%.*]] = select i1 [[TMP14]], i64 [[TMP13]], i64 [[TMP12]]
-; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[TMP15]], 1
-; CHECK-NEXT: [[TMP16:%.*]] = sub i64 [[LAST_ACTIVE_LANE]], 1
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[VEC_IND]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP16:%.*]] = sub i64 [[TMP15]], 1
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x i64> [[TMP27]], i64 [[TMP16]]
; CHECK-NEXT: [[TMP18:%.*]] = sub i64 [[TMP16]], 4
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i64> [[STEP_ADD]], i64 [[TMP18]]
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i64> [[TMP29]], i64 [[TMP18]]
; CHECK-NEXT: [[TMP20:%.*]] = icmp uge i64 [[TMP16]], 4
; CHECK-NEXT: [[TMP21:%.*]] = select i1 [[TMP20]], i64 [[TMP32]], i64 [[TMP17]]
; CHECK-NEXT: [[TMP22:%.*]] = sub i64 [[TMP16]], 8
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i64> [[STEP_ADD_2]], i64 [[TMP22]]
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i64> [[TMP31]], i64 [[TMP22]]
; CHECK-NEXT: [[TMP24:%.*]] = icmp uge i64 [[TMP16]], 8
; CHECK-NEXT: [[TMP25:%.*]] = select i1 [[TMP24]], i64 [[TMP23]], i64 [[TMP21]]
; CHECK-NEXT: [[TMP26:%.*]] = sub i64 [[TMP16]], 12
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i64> [[STEP_ADD_3]], i64 [[TMP26]]
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[TMP33]], i64 [[TMP26]]
; CHECK-NEXT: [[TMP28:%.*]] = icmp uge i64 [[TMP16]], 12
-; CHECK-NEXT: [[TMP29:%.*]] = select i1 [[TMP28]], i64 [[TMP27]], i64 [[TMP25]]
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i64> [[VECTOR_RECUR]], i32 3
-; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i64 [[LAST_ACTIVE_LANE]], 0
-; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP31]], i64 [[TMP30]], i64 [[TMP29]]
+; CHECK-NEXT: [[TMP19:%.*]] = select i1 [[TMP28]], i64 [[TMP30]], i64 [[TMP25]]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: exit:
; CHECK-NEXT: ret i64 [[TMP19]]
More information about the llvm-commits
mailing list