[llvm] [VPlan] Remove ComputeAnyOfResult, use ComputeReductionResult. (PR #190039)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 16 03:37:06 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/190039
>From b1cc5cb774ec738007b34cc96b5b7458875b1830 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Sun, 18 Jan 2026 12:56:50 +0000
Subject: [PATCH 1/5] [VPlan] Remove ComputeAnyOfResult, use
ComputeReductionResult.
ComputeAnyOfResult is simply a boolean OR reduction. Remove the
dedicated opcode and model directly via ComputeReductionResult.
This simplifies and unifies the code, as well as enabling trivial
constant folding.
---
.../Vectorize/LoopVectorizationPlanner.h | 8 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 36 ++++----
llvm/lib/Transforms/Vectorize/VPlan.h | 3 -
.../Transforms/Vectorize/VPlanAnalysis.cpp | 2 -
.../Transforms/Vectorize/VPlanPatternMatch.h | 7 --
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 23 -----
.../Transforms/Vectorize/VPlanTransforms.cpp | 9 +-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 2 -
.../LoopVectorize/X86/cost-model.ll | 2 -
.../X86/pr109581-unused-blend.ll | 4 +-
.../epilog-vectorization-any-of-reductions.ll | 14 ++--
.../epilog-vectorization-reductions.ll | 8 +-
.../LoopVectorize/select-cmp-multiuse.ll | 84 ++++++++-----------
13 files changed, 73 insertions(+), 129 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index bab9efee37533..7f685c6151a55 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -672,10 +672,10 @@ class LoopVectorizationPlanner {
/// legal to vectorize the loop. This method creates VPlans using VPRecipes.
void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF);
- /// Add recipes to compute the final reduction result (ComputeAnyOfResult,
- /// ComputeReductionResult depending on the reduction) in
- /// the middle block. Selects are introduced for reductions between the phi
- /// and users outside the vector region when folding the tail.
+ /// Add recipes to compute the final reduction result (using
+ /// ComputeReductionResult) in the middle block. Selects are introduced for
+ /// reductions between the phi and users outside the vector region when
+ /// folding the tail.
void addReductionResultComputation(VPlanPtr &Plan,
VPRecipeBuilder &RecipeBuilder,
ElementCount MinVF);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 193d33d554a94..3bc7a450c958b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8297,10 +8297,8 @@ void LoopVectorizationPlanner::addReductionResultComputation(
Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", *PhiR);
OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
using namespace VPlanPatternMatch;
- return match(
- &U, m_CombineOr(
- m_VPInstruction<VPInstruction::ComputeAnyOfResult>(),
- m_VPInstruction<VPInstruction::ComputeReductionResult>()));
+ return match(&U,
+ m_VPInstruction<VPInstruction::ComputeReductionResult>());
});
if (CM.usePredicatedReductionSelect(RecurrenceKind))
@@ -8324,7 +8322,7 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPBuilder::InsertPointGuard Guard(Builder);
Builder.setInsertPoint(MiddleVPBB, IP);
// For AnyOf reductions, find the select among PhiR's users. This is used
- // both to find NewVal for ComputeAnyOfResult and to adjust the reduction.
+ // both to find NewVal for the final result and to adjust the reduction.
VPRecipeBase *AnyOfSelect = nullptr;
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
AnyOfSelect = cast<VPRecipeBase>(*find_if(PhiR->users(), [](VPUser *U) {
@@ -8337,9 +8335,14 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPValue *NewVal = AnyOfSelect->getOperand(1) == PhiR
? AnyOfSelect->getOperand(2)
: AnyOfSelect->getOperand(1);
+ VPIRFlags Flags(RecurKind::Or, /*IsOrdered=*/false, /*IsInLoop=*/false,
+ FastMathFlags());
+ auto *OrReduce =
+ Builder.createNaryOp(VPInstruction::ComputeReductionResult,
+ {NewExitingVPV}, Flags, ExitDL);
+ auto *Freeze = Builder.createNaryOp(Instruction::Freeze, {OrReduce});
FinalReductionResult =
- Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
- {Start, NewVal, NewExitingVPV}, ExitDL);
+ Builder.createSelect(Freeze, NewVal, Start, {}, "rdx.select");
} else {
VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
PhiR->getFastMathFlags());
@@ -8385,11 +8388,9 @@ void LoopVectorizationPlanner::addReductionResultComputation(
auto *Parent = cast<VPRecipeBase>(U)->getParent();
if (FinalReductionResult == U || Parent->getParent())
continue;
- // Skip FindIV reduction chain recipes (ComputeReductionResult, icmp).
- if (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind) &&
- match(U, m_CombineOr(
- m_VPInstruction<VPInstruction::ComputeReductionResult>(),
- m_VPInstruction<Instruction::ICmp>())))
+ // Skip ComputeReductionResult when it is not the final result (e.g.
+ // for AnyOf or FindIV reductions).
+ if (match(U, m_VPInstruction<VPInstruction::ComputeReductionResult>()))
continue;
U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
@@ -8966,10 +8967,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
// value.
auto IsReductionResult = [](VPRecipeBase *R) {
auto *VPI = dyn_cast<VPInstruction>(R);
- if (!VPI)
- return false;
- return VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
- VPI->getOpcode() == VPInstruction::ComputeReductionResult;
+ return VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult;
};
auto *RdxResult = cast<VPInstruction>(
vputils::findRecipe(ReductionPhi->getBackedgeValue(), IsReductionResult));
@@ -8977,6 +8975,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
->getIncomingValueForBlock(L->getLoopPreheader());
+ RecurKind RK = ReductionPhi->getRecurrenceKind();
// Check for FindIV pattern by looking for icmp user of RdxResult.
// The pattern is: select(icmp ne RdxResult, Sentinel), RdxResult, Start
@@ -8988,8 +8987,9 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
m_VPValue(SentinelVPV)));
});
- if (RdxResult->getOpcode() == VPInstruction::ComputeAnyOfResult) {
- Value *StartV = RdxResult->getOperand(0)->getLiveInIRValue();
+ if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
+ Value *StartV = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
+ EPI.MainLoopIterationCountCheck);
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
// start value; compare the final value from the main vector loop
// to the start value.
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 21016f84d48a7..c150a283b2519 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1267,9 +1267,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// scalars extracted from a vector, to be replaced by VF ExtractElement
/// VPInstructions.
Unpack,
- /// Compute the final result of a AnyOf reduction with select(cmp(),x,y),
- /// where one of (x,y) is loop invariant, and both x and y are integer type.
- ComputeAnyOfResult,
ComputeReductionResult,
// Extracts the last part of its operand. Removed during unrolling.
ExtractLastPart,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
index 9667227a329ac..89224e611683f 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
@@ -106,8 +106,6 @@ Type *VPTypeAnalysis::inferScalarTypeForRecipe(const VPInstruction *R) {
inferScalarType(R->getOperand(1)) &&
"different types inferred for different operands");
return IntegerType::get(Ctx, 1);
- case VPInstruction::ComputeAnyOfResult:
- return inferScalarType(R->getOperand(1));
case VPInstruction::ExplicitVectorLength:
return Type::getIntNTy(Ctx, 32);
case VPInstruction::FirstOrderRecurrenceSplice:
diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index 2ce012c5a9f10..ea73615d157f9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -542,13 +542,6 @@ inline bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV, Op1_t Start)
m_ComputeReductionResult(ReducedIV), Start));
}
-template <typename Op0_t, typename Op1_t, typename Op2_t>
-inline VPInstruction_match<VPInstruction::ComputeAnyOfResult, Op0_t, Op1_t,
- Op2_t>
-m_ComputeAnyOfResult(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
- return m_VPInstruction<VPInstruction::ComputeAnyOfResult>(Op0, Op1, Op2);
-}
-
template <typename Op0_t>
inline VPInstruction_match<VPInstruction::Reverse, Op0_t>
m_Reverse(const Op0_t &Op0) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index b496278ebd6ec..f02eb067214bc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -501,7 +501,6 @@ unsigned VPInstruction::getNumOperandsForOpcode() const {
case VPInstruction::BuildStructVector:
case VPInstruction::BuildVector:
case VPInstruction::CanonicalIVIncrementForPart:
- case VPInstruction::ComputeAnyOfResult:
case VPInstruction::ComputeReductionResult:
case VPInstruction::FirstActiveLane:
case VPInstruction::LastActiveLane:
@@ -728,22 +727,6 @@ Value *VPInstruction::generate(VPTransformState &State) {
return Builder.CreateInsertElement(Iden, State.get(getOperand(0), true),
Builder.getInt32(0));
}
- case VPInstruction::ComputeAnyOfResult: {
- Value *Start = State.get(getOperand(0), VPLane(0));
- Value *NewVal = State.get(getOperand(1), VPLane(0));
- Value *ReducedResult = State.get(getOperand(2));
- for (unsigned Idx = 3; Idx < getNumOperands(); ++Idx)
- ReducedResult =
- Builder.CreateBinOp(Instruction::Or, State.get(getOperand(Idx)),
- ReducedResult, "bin.rdx");
- // If any predicate is true it means that we want to select the new value.
- if (ReducedResult->getType()->isVectorTy())
- ReducedResult = Builder.CreateOrReduce(ReducedResult);
- // The compares in the loop may yield poison, which propagates through the
- // bitwise ORs. Freeze it here before the condition is used.
- ReducedResult = Builder.CreateFreeze(ReducedResult);
- return Builder.CreateSelect(ReducedResult, NewVal, Start, "rdx.select");
- }
case VPInstruction::ComputeReductionResult: {
RecurKind RK = getRecurKind();
bool IsOrdered = isReductionOrdered();
@@ -1277,7 +1260,6 @@ bool VPInstruction::isVectorToScalar() const {
getOpcode() == VPInstruction::ExtractLane ||
getOpcode() == VPInstruction::FirstActiveLane ||
getOpcode() == VPInstruction::LastActiveLane ||
- getOpcode() == VPInstruction::ComputeAnyOfResult ||
getOpcode() == VPInstruction::ExtractLastActive ||
getOpcode() == VPInstruction::ComputeReductionResult ||
getOpcode() == VPInstruction::AnyOf;
@@ -1419,8 +1401,6 @@ bool VPInstruction::usesFirstLaneOnly(const VPValue *Op) const {
case VPInstruction::WidePtrAdd:
// WidePtrAdd supports scalar and vector base addresses.
return false;
- case VPInstruction::ComputeAnyOfResult:
- return Op == getOperand(0) || Op == getOperand(1);
case VPInstruction::ExitingIVValue:
case VPInstruction::ExtractLane:
return Op == getOperand(0);
@@ -1525,9 +1505,6 @@ void VPInstruction::printRecipe(raw_ostream &O, const Twine &Indent,
case VPInstruction::ExtractPenultimateElement:
O << "extract-penultimate-element";
break;
- case VPInstruction::ComputeAnyOfResult:
- O << "compute-anyof-result";
- break;
case VPInstruction::ComputeReductionResult:
O << "compute-reduction-result";
break;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 207099422905d..198b4ce9fe71a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5840,9 +5840,14 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
VPValue *OrVal = LoopBuilder.createOr(AnyOfPhi, AnyOfCond);
AnyOfPhi->setOperand(1, OrVal);
+ VPIRFlags OrFlags(RecurKind::Or, /*IsOrdered=*/false,
+ /*IsInLoop=*/false, FastMathFlags());
+ auto *OrReduce = MiddleBuilder.createNaryOp(
+ VPInstruction::ComputeReductionResult, {OrVal}, OrFlags, ExitDL);
+ auto *Freeze =
+ MiddleBuilder.createNaryOp(Instruction::Freeze, {OrReduce});
NewRdxResult =
- MiddleBuilder.createNaryOp(VPInstruction::ComputeAnyOfResult,
- {StartVPV, ReducedIV, OrVal}, {}, ExitDL);
+ MiddleBuilder.createSelect(Freeze, ReducedIV, StartVPV, ExitDL);
// Initialize the IV reduction phi with the neutral element, not the
// original start value, to ensure correct min/max reduction results.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 50b05a9e9c0fd..b6ea7365a9a8b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -428,8 +428,6 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
if (match(&R, m_VPInstruction<VPInstruction::AnyOf>(m_VPValue(Op1))) ||
match(&R, m_FirstActiveLane(m_VPValue(Op1))) ||
match(&R, m_LastActiveLane(m_VPValue(Op1))) ||
- match(&R,
- m_ComputeAnyOfResult(m_VPValue(), m_VPValue(), m_VPValue(Op1))) ||
match(&R, m_ComputeReductionResult(m_VPValue(Op1)))) {
addUniformForAllParts(cast<VPInstruction>(&R));
for (unsigned Part = 1; Part != UF; ++Part)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index 77f305b7115de..eab7b9ba76a8c 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -459,8 +459,6 @@ define i1 @any_of_cost(ptr %start, ptr %end) #0 {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <2 x i1> [[TMP27]], [[TMP26]]
; CHECK-NEXT: [[TMP29:%.*]] = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> [[BIN_RDX]])
-; CHECK-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP29]]
-; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP30]], i1 false, i1 false
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
index 36ad765993588..abbbbe0f1ad01 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
@@ -71,13 +71,11 @@ define i32 @unused_blend_after_unrolling(ptr %p, i32 %a, i1 %c.1, i16 %x, i16 %y
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP25]], [[TMP24]]
; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX]])
-; CHECK-NEXT: [[TMP28:%.*]] = freeze i1 [[TMP27]]
-; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP28]], i32 0, i32 0
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[B:%.*]] = phi i32 [ [[RDX_SELECT]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[B:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 97, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH]] ]
; CHECK-NEXT: br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[THEN:.*]]
; CHECK: [[THEN]]:
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
index e5377cbfc35d3..8fea4fdb7d98c 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
@@ -220,8 +220,6 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
-; CHECK-NEXT: [[TMP6:%.*]] = freeze i1 [[TMP5]]
-; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP6]], i1 false, i1 false
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
@@ -229,7 +227,7 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ false, [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i1 [[BC_MERGE_RDX]], false
; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[TMP0]], 4
@@ -255,13 +253,11 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
-; CHECK-NEXT: [[TMP13:%.*]] = freeze i1 [[TMP12]]
-; CHECK-NEXT: [[RDX_SELECT16:%.*]] = select i1 [[TMP13]], i1 false, i1 false
; CHECK-NEXT: [[CMP_N8:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[CMP_N8]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i1 [ [[RDX_SELECT16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i1 [ false, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ false, [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[ITER_CHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
@@ -275,7 +271,7 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
; CHECK-NEXT: [[CMP_2:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[CMP_2]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: exit:
-; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i1 [ [[SEL]], [[LOOP]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ [[RDX_SELECT16]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i1 [ [[SEL]], [[LOOP]] ], [ false, [[MIDDLE_BLOCK]] ], [ false, [[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i1 [[SEL_LCSSA]]
;
entry:
@@ -353,7 +349,7 @@ define i1 @any_of_reduction_i1_epilog2(ptr %start, ptr %end, i64 %x) {
; CHECK: middle.block:
; CHECK-NEXT: [[TMP23:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[RDX_SELECT_CMP]])
; CHECK-NEXT: [[TMP47:%.*]] = freeze i1 [[TMP23]]
-; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP47]], i1 false, i1 true
+; CHECK-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP47]], true
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
@@ -403,7 +399,7 @@ define i1 @any_of_reduction_i1_epilog2(ptr %start, ptr %end, i64 %x) {
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: [[TMP49:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP43]])
; CHECK-NEXT: [[TMP45:%.*]] = freeze i1 [[TMP49]]
-; CHECK-NEXT: [[RDX_SELECT22:%.*]] = select i1 [[TMP45]], i1 false, i1 true
+; CHECK-NEXT: [[RDX_SELECT22:%.*]] = xor i1 [[TMP45]], true
; CHECK-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC8]]
; CHECK-NEXT: br i1 [[CMP_N10]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index f08f7ac0fd4e2..343873e464e03 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -1140,8 +1140,6 @@ define i32 @test_foldable_reduction(i64 %N) {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[VEC_PHI]])
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
-; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP4]], i32 0, i32 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; CHECK: [[VEC_EPILOG_ITER_CHECK]]:
@@ -1163,14 +1161,12 @@ define i32 @test_foldable_reduction(i64 %N) {
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[VEC_PHI6]])
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP8]]
-; CHECK-NEXT: [[RDX_SELECT8:%.*]] = select i1 [[TMP9]], i32 0, i32 0
; CHECK-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC4]]
; CHECK-NEXT: br i1 [[CMP_N9]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
; CHECK: [[VEC_EPILOG_SCALAR_PH]]:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[TMP7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP2]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i32 [ [[RDX_SELECT8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[RDX_SELECT]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i32 [ 0, %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ 0, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -1183,7 +1179,7 @@ define i32 @test_foldable_reduction(i64 %N) {
; CHECK-NEXT: [[EC:%.*]] = icmp ne i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[EC]], label %[[LOOP]], label %[[EXIT]], !llvm.loop [[LOOP41:![0-9]+]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[R_1:%.*]] = phi i32 [ [[RED_2_NEXT]], %[[LOOP]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ [[RDX_SELECT8]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[R_1:%.*]] = phi i32 [ [[RED_2_NEXT]], %[[LOOP]] ], [ 0, %[[MIDDLE_BLOCK]] ], [ 0, %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[R_2:%.*]] = phi i32 [ [[RED_1_NEXT]], %[[LOOP]] ], [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[TMP7]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: [[R:%.*]] = add i32 [[R_1]], [[R_2]]
; CHECK-NEXT: ret i32 [[R]]
diff --git a/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll b/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
index ffb16d07f3ea8..5d6b2c6a9e729 100644
--- a/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
+++ b/llvm/test/Transforms/LoopVectorize/select-cmp-multiuse.ll
@@ -42,16 +42,15 @@ define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
; CHECK-VF4-IC1: middle.block:
; CHECK-VF4-IC1-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
; CHECK-VF4-IC1-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP8]]
-; CHECK-VF4-IC1-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP9]], i1 false, i1 true
+; CHECK-VF4-IC1-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP9]], true
; CHECK-VF4-IC1-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
; CHECK-VF4-IC1-NEXT: [[TMP11:%.*]] = freeze i1 [[TMP10]]
-; CHECK-VF4-IC1-NEXT: [[RDX_SELECT2:%.*]] = select i1 [[TMP11]], i1 true, i1 false
; CHECK-VF4-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF4-IC1-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF4-IC1: scalar.ph:
; CHECK-VF4-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i1 [ [[RDX_SELECT2]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i1 [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
; CHECK-VF4-IC1-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF4-IC1: for.body:
; CHECK-VF4-IC1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -66,7 +65,7 @@ define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
; CHECK-VF4-IC1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
; CHECK-VF4-IC1-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-VF4-IC1: exit:
-; CHECK-VF4-IC1-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[RDX_SELECT2]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4-IC1-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC1-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC1-NEXT: [[TMP12:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
; CHECK-VF4-IC1-NEXT: [[TMP13:%.*]] = select i1 [[ALL_0_OFF0__LCSSA]], i32 1, i32 [[TMP12]]
@@ -106,17 +105,16 @@ define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
; CHECK-VF4-IC2-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP13]], [[TMP12]]
; CHECK-VF4-IC2-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX]])
; CHECK-VF4-IC2-NEXT: [[TMP16:%.*]] = freeze i1 [[TMP15]]
-; CHECK-VF4-IC2-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP16]], i1 false, i1 true
+; CHECK-VF4-IC2-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP16]], true
; CHECK-VF4-IC2-NEXT: [[BIN_RDX5:%.*]] = or <4 x i1> [[TMP9]], [[TMP8]]
; CHECK-VF4-IC2-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX5]])
; CHECK-VF4-IC2-NEXT: [[TMP18:%.*]] = freeze i1 [[TMP17]]
-; CHECK-VF4-IC2-NEXT: [[RDX_SELECT6:%.*]] = select i1 [[TMP18]], i1 true, i1 false
; CHECK-VF4-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF4-IC2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF4-IC2: scalar.ph:
; CHECK-VF4-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i1 [ [[RDX_SELECT6]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i1 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
; CHECK-VF4-IC2-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF4-IC2: for.body:
; CHECK-VF4-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -131,7 +129,7 @@ define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
; CHECK-VF4-IC2-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
; CHECK-VF4-IC2-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-VF4-IC2: exit:
-; CHECK-VF4-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[RDX_SELECT6]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC2-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC2-NEXT: [[TMP19:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
; CHECK-VF4-IC2-NEXT: [[TMP20:%.*]] = select i1 [[ALL_0_OFF0__LCSSA]], i32 1, i32 [[TMP19]]
@@ -171,16 +169,15 @@ define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
; CHECK-VF1-IC2: middle.block:
; CHECK-VF1-IC2-NEXT: [[BIN_RDX:%.*]] = or i1 [[TMP13]], [[TMP12]]
; CHECK-VF1-IC2-NEXT: [[TMP15:%.*]] = freeze i1 [[BIN_RDX]]
-; CHECK-VF1-IC2-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP15]], i1 false, i1 true
+; CHECK-VF1-IC2-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP15]], true
; CHECK-VF1-IC2-NEXT: [[BIN_RDX4:%.*]] = or i1 [[TMP9]], [[TMP8]]
; CHECK-VF1-IC2-NEXT: [[TMP16:%.*]] = freeze i1 [[BIN_RDX4]]
-; CHECK-VF1-IC2-NEXT: [[RDX_SELECT5:%.*]] = select i1 [[TMP16]], i1 true, i1 false
; CHECK-VF1-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF1-IC2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF1-IC2: scalar.ph:
; CHECK-VF1-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i1 [ [[RDX_SELECT5]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i1 [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
; CHECK-VF1-IC2-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF1-IC2: for.body:
; CHECK-VF1-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -195,7 +192,7 @@ define i32 @multi_user_cmp(ptr readonly %a, i64 noundef %n) {
; CHECK-VF1-IC2-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
; CHECK-VF1-IC2-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-VF1-IC2: exit:
-; CHECK-VF1-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[RDX_SELECT5]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF1-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
; CHECK-VF1-IC2-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF1-IC2-NEXT: [[TMP17:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
; CHECK-VF1-IC2-NEXT: [[TMP18:%.*]] = select i1 [[ALL_0_OFF0__LCSSA]], i32 1, i32 [[TMP17]]
@@ -261,16 +258,15 @@ define i32 @multi_user_cmp_int(ptr readonly %a, i64 noundef %n) {
; CHECK-VF4-IC1: middle.block:
; CHECK-VF4-IC1-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
; CHECK-VF4-IC1-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP8]]
-; CHECK-VF4-IC1-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP9]], i1 false, i1 true
+; CHECK-VF4-IC1-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP9]], true
; CHECK-VF4-IC1-NEXT: [[TMP10:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
; CHECK-VF4-IC1-NEXT: [[TMP11:%.*]] = freeze i1 [[TMP10]]
-; CHECK-VF4-IC1-NEXT: [[RDX_SELECT2:%.*]] = select i1 [[TMP11]], i1 true, i1 false
; CHECK-VF4-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF4-IC1-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF4-IC1: scalar.ph:
; CHECK-VF4-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i1 [ [[RDX_SELECT2]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i1 [ [[TMP11]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
; CHECK-VF4-IC1-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF4-IC1: for.body:
; CHECK-VF4-IC1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -285,7 +281,7 @@ define i32 @multi_user_cmp_int(ptr readonly %a, i64 noundef %n) {
; CHECK-VF4-IC1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
; CHECK-VF4-IC1-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-VF4-IC1: exit:
-; CHECK-VF4-IC1-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[RDX_SELECT2]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4-IC1-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC1-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC1-NEXT: [[TMP12:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
; CHECK-VF4-IC1-NEXT: [[TMP13:%.*]] = select i1 [[ALL_0_OFF0__LCSSA]], i32 1, i32 [[TMP12]]
@@ -325,17 +321,16 @@ define i32 @multi_user_cmp_int(ptr readonly %a, i64 noundef %n) {
; CHECK-VF4-IC2-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP13]], [[TMP12]]
; CHECK-VF4-IC2-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX]])
; CHECK-VF4-IC2-NEXT: [[TMP16:%.*]] = freeze i1 [[TMP15]]
-; CHECK-VF4-IC2-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP16]], i1 false, i1 true
+; CHECK-VF4-IC2-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP16]], true
; CHECK-VF4-IC2-NEXT: [[BIN_RDX5:%.*]] = or <4 x i1> [[TMP9]], [[TMP8]]
; CHECK-VF4-IC2-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX5]])
; CHECK-VF4-IC2-NEXT: [[TMP18:%.*]] = freeze i1 [[TMP17]]
-; CHECK-VF4-IC2-NEXT: [[RDX_SELECT6:%.*]] = select i1 [[TMP18]], i1 true, i1 false
; CHECK-VF4-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF4-IC2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF4-IC2: scalar.ph:
; CHECK-VF4-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i1 [ [[RDX_SELECT6]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i1 [ [[TMP18]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
; CHECK-VF4-IC2-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF4-IC2: for.body:
; CHECK-VF4-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -350,7 +345,7 @@ define i32 @multi_user_cmp_int(ptr readonly %a, i64 noundef %n) {
; CHECK-VF4-IC2-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
; CHECK-VF4-IC2-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-VF4-IC2: exit:
-; CHECK-VF4-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[RDX_SELECT6]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[TMP18]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC2-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC2-NEXT: [[TMP19:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
; CHECK-VF4-IC2-NEXT: [[TMP20:%.*]] = select i1 [[ALL_0_OFF0__LCSSA]], i32 1, i32 [[TMP19]]
@@ -390,16 +385,15 @@ define i32 @multi_user_cmp_int(ptr readonly %a, i64 noundef %n) {
; CHECK-VF1-IC2: middle.block:
; CHECK-VF1-IC2-NEXT: [[BIN_RDX:%.*]] = or i1 [[TMP13]], [[TMP12]]
; CHECK-VF1-IC2-NEXT: [[TMP15:%.*]] = freeze i1 [[BIN_RDX]]
-; CHECK-VF1-IC2-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP15]], i1 false, i1 true
+; CHECK-VF1-IC2-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP15]], true
; CHECK-VF1-IC2-NEXT: [[BIN_RDX4:%.*]] = or i1 [[TMP9]], [[TMP8]]
; CHECK-VF1-IC2-NEXT: [[TMP16:%.*]] = freeze i1 [[BIN_RDX4]]
-; CHECK-VF1-IC2-NEXT: [[RDX_SELECT5:%.*]] = select i1 [[TMP16]], i1 true, i1 false
; CHECK-VF1-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF1-IC2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF1-IC2: scalar.ph:
; CHECK-VF1-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i1 [ [[RDX_SELECT5]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i1 [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
; CHECK-VF1-IC2-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF1-IC2: for.body:
; CHECK-VF1-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -414,7 +408,7 @@ define i32 @multi_user_cmp_int(ptr readonly %a, i64 noundef %n) {
; CHECK-VF1-IC2-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
; CHECK-VF1-IC2-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-VF1-IC2: exit:
-; CHECK-VF1-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[RDX_SELECT5]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF1-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
; CHECK-VF1-IC2-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF1-IC2-NEXT: [[TMP17:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
; CHECK-VF1-IC2-NEXT: [[TMP18:%.*]] = select i1 [[ALL_0_OFF0__LCSSA]], i32 1, i32 [[TMP17]]
@@ -530,16 +524,15 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
; CHECK-VF4-IC1: middle.block:
; CHECK-VF4-IC1-NEXT: [[TMP32:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]])
; CHECK-VF4-IC1-NEXT: [[TMP33:%.*]] = freeze i1 [[TMP32]]
-; CHECK-VF4-IC1-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP33]], i1 false, i1 true
+; CHECK-VF4-IC1-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP33]], true
; CHECK-VF4-IC1-NEXT: [[TMP34:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]])
; CHECK-VF4-IC1-NEXT: [[TMP35:%.*]] = freeze i1 [[TMP34]]
-; CHECK-VF4-IC1-NEXT: [[RDX_SELECT9:%.*]] = select i1 [[TMP35]], i1 true, i1 false
; CHECK-VF4-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF4-IC1-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF4-IC1: scalar.ph:
; CHECK-VF4-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ], [ true, [[VECTOR_MEMCHECK]] ]
-; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i1 [ [[RDX_SELECT9]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i1 [ [[TMP35]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
; CHECK-VF4-IC1-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF4-IC1: for.body:
; CHECK-VF4-IC1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END6:%.*]] ]
@@ -562,7 +555,7 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
; CHECK-VF4-IC1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
; CHECK-VF4-IC1-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-VF4-IC1: exit:
-; CHECK-VF4-IC1-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[IF_END6]] ], [ [[RDX_SELECT9]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4-IC1-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[IF_END6]] ], [ [[TMP35]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC1-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[IF_END6]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC1-NEXT: [[TMP36:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
; CHECK-VF4-IC1-NEXT: [[TMP37:%.*]] = select i1 [[ALL_0_OFF0__LCSSA]], i32 1, i32 [[TMP36]]
@@ -689,17 +682,16 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
; CHECK-VF4-IC2-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP14]], [[TMP13]]
; CHECK-VF4-IC2-NEXT: [[TMP62:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX]])
; CHECK-VF4-IC2-NEXT: [[TMP63:%.*]] = freeze i1 [[TMP62]]
-; CHECK-VF4-IC2-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP63]], i1 false, i1 true
+; CHECK-VF4-IC2-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP63]], true
; CHECK-VF4-IC2-NEXT: [[BIN_RDX20:%.*]] = or <4 x i1> [[TMP10]], [[TMP9]]
; CHECK-VF4-IC2-NEXT: [[TMP64:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX20]])
; CHECK-VF4-IC2-NEXT: [[TMP65:%.*]] = freeze i1 [[TMP64]]
-; CHECK-VF4-IC2-NEXT: [[RDX_SELECT21:%.*]] = select i1 [[TMP65]], i1 true, i1 false
; CHECK-VF4-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF4-IC2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF4-IC2: scalar.ph:
; CHECK-VF4-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ], [ true, [[VECTOR_MEMCHECK]] ]
-; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX22:%.*]] = phi i1 [ [[RDX_SELECT21]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX22:%.*]] = phi i1 [ [[TMP65]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
; CHECK-VF4-IC2-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF4-IC2: for.body:
; CHECK-VF4-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END6:%.*]] ]
@@ -722,7 +714,7 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
; CHECK-VF4-IC2-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
; CHECK-VF4-IC2-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-VF4-IC2: exit:
-; CHECK-VF4-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[IF_END6]] ], [ [[RDX_SELECT21]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[IF_END6]] ], [ [[TMP65]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC2-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[IF_END6]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC2-NEXT: [[TMP66:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
; CHECK-VF4-IC2-NEXT: [[TMP67:%.*]] = select i1 [[ALL_0_OFF0__LCSSA]], i32 1, i32 [[TMP66]]
@@ -786,16 +778,15 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
; CHECK-VF1-IC2: middle.block:
; CHECK-VF1-IC2-NEXT: [[BIN_RDX:%.*]] = or i1 [[TMP14]], [[TMP13]]
; CHECK-VF1-IC2-NEXT: [[TMP24:%.*]] = freeze i1 [[BIN_RDX]]
-; CHECK-VF1-IC2-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP24]], i1 false, i1 true
+; CHECK-VF1-IC2-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP24]], true
; CHECK-VF1-IC2-NEXT: [[BIN_RDX7:%.*]] = or i1 [[TMP10]], [[TMP9]]
; CHECK-VF1-IC2-NEXT: [[TMP25:%.*]] = freeze i1 [[BIN_RDX7]]
-; CHECK-VF1-IC2-NEXT: [[RDX_SELECT8:%.*]] = select i1 [[TMP25]], i1 true, i1 false
; CHECK-VF1-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF1-IC2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF1-IC2: scalar.ph:
; CHECK-VF1-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ], [ true, [[VECTOR_MEMCHECK]] ]
-; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX9:%.*]] = phi i1 [ [[RDX_SELECT8]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
+; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX9:%.*]] = phi i1 [ [[TMP25]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ], [ false, [[VECTOR_MEMCHECK]] ]
; CHECK-VF1-IC2-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF1-IC2: for.body:
; CHECK-VF1-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END6:%.*]] ]
@@ -818,7 +809,7 @@ define i32 @multi_user_cmp_branch_use(ptr readonly %a, ptr %b, i64 noundef %n) {
; CHECK-VF1-IC2-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
; CHECK-VF1-IC2-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK-VF1-IC2: exit:
-; CHECK-VF1-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[IF_END6]] ], [ [[RDX_SELECT8]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF1-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[IF_END6]] ], [ [[TMP25]], [[MIDDLE_BLOCK]] ]
; CHECK-VF1-IC2-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[IF_END6]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF1-IC2-NEXT: [[TMP26:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
; CHECK-VF1-IC2-NEXT: [[TMP27:%.*]] = select i1 [[ALL_0_OFF0__LCSSA]], i32 1, i32 [[TMP26]]
@@ -896,17 +887,16 @@ define i32 @multi_user_cmp_branch_use_and_outside_bb_use(ptr readonly %a, i64 no
; CHECK-VF4-IC1: middle.block:
; CHECK-VF4-IC1-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]])
; CHECK-VF4-IC1-NEXT: [[TMP10:%.*]] = freeze i1 [[TMP9]]
-; CHECK-VF4-IC1-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP10]], i1 false, i1 true
+; CHECK-VF4-IC1-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP10]], true
; CHECK-VF4-IC1-NEXT: [[TMP11:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]])
; CHECK-VF4-IC1-NEXT: [[TMP12:%.*]] = freeze i1 [[TMP11]]
-; CHECK-VF4-IC1-NEXT: [[RDX_SELECT2:%.*]] = select i1 [[TMP12]], i1 true, i1 false
; CHECK-VF4-IC1-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP3]], i32 3
; CHECK-VF4-IC1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF4-IC1-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF4-IC1: scalar.ph:
; CHECK-VF4-IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i1 [ [[RDX_SELECT2]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF4-IC1-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i1 [ [[TMP12]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
; CHECK-VF4-IC1-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF4-IC1: for.body:
; CHECK-VF4-IC1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -922,7 +912,7 @@ define i32 @multi_user_cmp_branch_use_and_outside_bb_use(ptr readonly %a, i64 no
; CHECK-VF4-IC1-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK-VF4-IC1: exit:
; CHECK-VF4-IC1-NEXT: [[CMP1_LCSSA:%.*]] = phi i1 [ [[CMP1]], [[FOR_BODY]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
-; CHECK-VF4-IC1-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[RDX_SELECT2]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4-IC1-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC1-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC1-NEXT: [[TMP13:%.*]] = zext i1 [[CMP1_LCSSA]] to i32
; CHECK-VF4-IC1-NEXT: [[TMP14:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
@@ -963,18 +953,17 @@ define i32 @multi_user_cmp_branch_use_and_outside_bb_use(ptr readonly %a, i64 no
; CHECK-VF4-IC2-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP13]], [[TMP12]]
; CHECK-VF4-IC2-NEXT: [[TMP16:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX]])
; CHECK-VF4-IC2-NEXT: [[TMP17:%.*]] = freeze i1 [[TMP16]]
-; CHECK-VF4-IC2-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP17]], i1 false, i1 true
+; CHECK-VF4-IC2-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP17]], true
; CHECK-VF4-IC2-NEXT: [[BIN_RDX5:%.*]] = or <4 x i1> [[TMP9]], [[TMP8]]
; CHECK-VF4-IC2-NEXT: [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX5]])
; CHECK-VF4-IC2-NEXT: [[TMP19:%.*]] = freeze i1 [[TMP18]]
-; CHECK-VF4-IC2-NEXT: [[RDX_SELECT6:%.*]] = select i1 [[TMP19]], i1 true, i1 false
; CHECK-VF4-IC2-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP7]], i32 3
; CHECK-VF4-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF4-IC2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF4-IC2: scalar.ph:
; CHECK-VF4-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i1 [ [[RDX_SELECT6]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF4-IC2-NEXT: [[BC_MERGE_RDX7:%.*]] = phi i1 [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
; CHECK-VF4-IC2-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF4-IC2: for.body:
; CHECK-VF4-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -990,7 +979,7 @@ define i32 @multi_user_cmp_branch_use_and_outside_bb_use(ptr readonly %a, i64 no
; CHECK-VF4-IC2-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK-VF4-IC2: exit:
; CHECK-VF4-IC2-NEXT: [[CMP1_LCSSA:%.*]] = phi i1 [ [[CMP1]], [[FOR_BODY]] ], [ [[TMP15]], [[MIDDLE_BLOCK]] ]
-; CHECK-VF4-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[RDX_SELECT6]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF4-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC2-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF4-IC2-NEXT: [[TMP20:%.*]] = zext i1 [[CMP1_LCSSA]] to i32
; CHECK-VF4-IC2-NEXT: [[TMP21:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
@@ -1031,16 +1020,15 @@ define i32 @multi_user_cmp_branch_use_and_outside_bb_use(ptr readonly %a, i64 no
; CHECK-VF1-IC2: middle.block:
; CHECK-VF1-IC2-NEXT: [[BIN_RDX:%.*]] = or i1 [[TMP13]], [[TMP12]]
; CHECK-VF1-IC2-NEXT: [[TMP15:%.*]] = freeze i1 [[BIN_RDX]]
-; CHECK-VF1-IC2-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP15]], i1 false, i1 true
+; CHECK-VF1-IC2-NEXT: [[RDX_SELECT:%.*]] = xor i1 [[TMP15]], true
; CHECK-VF1-IC2-NEXT: [[BIN_RDX4:%.*]] = or i1 [[TMP9]], [[TMP8]]
; CHECK-VF1-IC2-NEXT: [[TMP16:%.*]] = freeze i1 [[BIN_RDX4]]
-; CHECK-VF1-IC2-NEXT: [[RDX_SELECT5:%.*]] = select i1 [[TMP16]], i1 true, i1 false
; CHECK-VF1-IC2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-VF1-IC2-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]]
; CHECK-VF1-IC2: scalar.ph:
; CHECK-VF1-IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ], [ true, [[ENTRY]] ]
-; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i1 [ [[RDX_SELECT5]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
+; CHECK-VF1-IC2-NEXT: [[BC_MERGE_RDX6:%.*]] = phi i1 [ [[TMP16]], [[MIDDLE_BLOCK]] ], [ false, [[ENTRY]] ]
; CHECK-VF1-IC2-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-VF1-IC2: for.body:
; CHECK-VF1-IC2-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -1056,7 +1044,7 @@ define i32 @multi_user_cmp_branch_use_and_outside_bb_use(ptr readonly %a, i64 no
; CHECK-VF1-IC2-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK-VF1-IC2: exit:
; CHECK-VF1-IC2-NEXT: [[CMP1_LCSSA:%.*]] = phi i1 [ [[CMP1]], [[FOR_BODY]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
-; CHECK-VF1-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[RDX_SELECT5]], [[MIDDLE_BLOCK]] ]
+; CHECK-VF1-IC2-NEXT: [[DOTANY_0_OFF0_LCSSA:%.*]] = phi i1 [ [[DOTANY_0_OFF0]], [[FOR_BODY]] ], [ [[TMP16]], [[MIDDLE_BLOCK]] ]
; CHECK-VF1-IC2-NEXT: [[ALL_0_OFF0__LCSSA:%.*]] = phi i1 [ [[ALL_0_OFF0_]], [[FOR_BODY]] ], [ [[RDX_SELECT]], [[MIDDLE_BLOCK]] ]
; CHECK-VF1-IC2-NEXT: [[TMP17:%.*]] = zext i1 [[CMP1_LCSSA]] to i32
; CHECK-VF1-IC2-NEXT: [[TMP18:%.*]] = select i1 [[DOTANY_0_OFF0_LCSSA]], i32 2, i32 3
>From 42d46f71783564dbd2530bd727fb6064b862f3fb Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 8 Apr 2026 18:43:14 +0100
Subject: [PATCH 2/5] [VPlan] Handle AnyOf Or reduction via
ComputeReductionResult.
Instead of having ComputeAnyOfResult handle the Or reduction of
unrolled parts inline, route it through ComputeReductionResult with
RecurKind::Or. ComputeAnyOfResult now takes a pre-reduced scalar
and only performs the freeze + select.
This is a preparatory step towards removing ComputeAnyOfResult
entirely in https://github.com/llvm/llvm-project/pull/190039.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 32 +++++++++++++------
.../lib/Transforms/Vectorize/VPlanRecipes.cpp | 9 +-----
.../Transforms/Vectorize/VPlanTransforms.cpp | 6 +++-
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 7 ++--
.../epilog-vectorization-reductions.ll | 4 +--
5 files changed, 35 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1fc880208ebf7..10cca6377bb56 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8368,9 +8368,13 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPValue *NewVal = AnyOfSelect->getOperand(1) == PhiR
? AnyOfSelect->getOperand(2)
: AnyOfSelect->getOperand(1);
- FinalReductionResult =
- Builder.createNaryOp(VPInstruction::ComputeAnyOfResult,
- {Start, NewVal, NewExitingVPV}, ExitDL);
+ VPIRFlags OrFlags(RecurKind::Or, /*IsOrdered=*/false,
+ /*IsInLoop=*/false, FastMathFlags());
+ auto *OrReduce =
+ Builder.createNaryOp(VPInstruction::ComputeReductionResult,
+ {NewExitingVPV}, OrFlags, ExitDL);
+ FinalReductionResult = Builder.createNaryOp(
+ VPInstruction::ComputeAnyOfResult, {Start, NewVal, OrReduce}, ExitDL);
} else {
VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
PhiR->getFastMathFlags());
@@ -8416,11 +8420,11 @@ void LoopVectorizationPlanner::addReductionResultComputation(
auto *Parent = cast<VPRecipeBase>(U)->getParent();
if (FinalReductionResult == U || Parent->getParent())
continue;
- // Skip FindIV reduction chain recipes (ComputeReductionResult, icmp).
- if (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind) &&
- match(U, m_CombineOr(
- m_VPInstruction<VPInstruction::ComputeReductionResult>(),
- m_VPInstruction<Instruction::ICmp>())))
+ // Skip ComputeReductionResult and FindIV reductions when they are not the
+ // final result.
+ if (match(U, m_VPInstruction<VPInstruction::ComputeReductionResult>()) ||
+ (RecurrenceDescriptor::isFindIVRecurrenceKind(RecurrenceKind) &&
+ match(U, m_VPInstruction<Instruction::ICmp>())))
continue;
U->replaceUsesOfWith(OrigExitingVPV, FinalReductionResult);
@@ -8995,12 +8999,20 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(&R)) {
// Find the reduction result by searching users of the phi or its backedge
// value.
+ using namespace VPlanPatternMatch;
auto IsReductionResult = [](VPRecipeBase *R) {
auto *VPI = dyn_cast<VPInstruction>(R);
if (!VPI)
return false;
- return VPI->getOpcode() == VPInstruction::ComputeAnyOfResult ||
- VPI->getOpcode() == VPInstruction::ComputeReductionResult;
+ // Prefer ComputeAnyOfResult over intermediate ComputeReductionResult,
+ // which is used for the Or reduction in AnyOf reductions.
+ if (VPI->getOpcode() == VPInstruction::ComputeAnyOfResult)
+ return true;
+ if (VPI->getOpcode() != VPInstruction::ComputeReductionResult)
+ return false;
+ return !any_of(VPI->users(), [](VPUser *U) {
+ return match(U, m_VPInstruction<VPInstruction::ComputeAnyOfResult>());
+ });
};
auto *RdxResult = cast<VPInstruction>(
vputils::findRecipe(ReductionPhi->getBackedgeValue(), IsReductionResult));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index bc442c22d1256..2c2f3915c41d4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -723,14 +723,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
case VPInstruction::ComputeAnyOfResult: {
Value *Start = State.get(getOperand(0), VPLane(0));
Value *NewVal = State.get(getOperand(1), VPLane(0));
- Value *ReducedResult = State.get(getOperand(2));
- for (unsigned Idx = 3; Idx < getNumOperands(); ++Idx)
- ReducedResult =
- Builder.CreateBinOp(Instruction::Or, State.get(getOperand(Idx)),
- ReducedResult, "bin.rdx");
- // If any predicate is true it means that we want to select the new value.
- if (ReducedResult->getType()->isVectorTy())
- ReducedResult = Builder.CreateOrReduce(ReducedResult);
+ Value *ReducedResult = State.get(getOperand(2), VPLane(0));
// The compares in the loop may yield poison, which propagates through the
// bitwise ORs. Freeze it here before the condition is used.
ReducedResult = Builder.CreateFreeze(ReducedResult);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 4264a216e2455..ef8c88ad81785 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5947,9 +5947,13 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
VPValue *OrVal = LoopBuilder.createOr(AnyOfPhi, AnyOfCond);
AnyOfPhi->setOperand(1, OrVal);
+ VPIRFlags OrFlags(RecurKind::Or, /*IsOrdered=*/false,
+ /*IsInLoop=*/false, FastMathFlags());
+ auto *OrReduce = MiddleBuilder.createNaryOp(
+ VPInstruction::ComputeReductionResult, {OrVal}, OrFlags, ExitDL);
NewRdxResult = MiddleBuilder.createNaryOp(
VPInstruction::ComputeAnyOfResult,
- {StartVPV, VectorRegionExitingVal, OrVal}, {}, ExitDL);
+ {StartVPV, VectorRegionExitingVal, OrReduce}, {}, ExitDL);
// Initialize the IV reduction phi with the neutral element, not the
// original start value, to ensure correct min/max reduction results.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index 50b05a9e9c0fd..ef5fef7a75fe0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -428,14 +428,17 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
if (match(&R, m_VPInstruction<VPInstruction::AnyOf>(m_VPValue(Op1))) ||
match(&R, m_FirstActiveLane(m_VPValue(Op1))) ||
match(&R, m_LastActiveLane(m_VPValue(Op1))) ||
- match(&R,
- m_ComputeAnyOfResult(m_VPValue(), m_VPValue(), m_VPValue(Op1))) ||
match(&R, m_ComputeReductionResult(m_VPValue(Op1)))) {
addUniformForAllParts(cast<VPInstruction>(&R));
for (unsigned Part = 1; Part != UF; ++Part)
R.addOperand(getValueForPart(Op1, Part));
continue;
}
+ if (match(&R,
+ m_ComputeAnyOfResult(m_VPValue(), m_VPValue(), m_VPValue()))) {
+ addUniformForAllParts(cast<VPInstruction>(&R));
+ continue;
+ }
VPValue *Op0;
if (match(&R, m_ExtractLane(m_VPValue(Op0), m_VPValue(Op1)))) {
addUniformForAllParts(cast<VPInstruction>(&R));
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
index f08f7ac0fd4e2..43487c0c87e04 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-reductions.ll
@@ -1140,7 +1140,7 @@ define i32 @test_foldable_reduction(i64 %N) {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP2:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[VEC_PHI]])
; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = freeze i1 true
; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP4]], i32 0, i32 0
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
@@ -1163,7 +1163,7 @@ define i32 @test_foldable_reduction(i64 %N) {
; CHECK: [[VEC_EPILOG_MIDDLE_BLOCK]]:
; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.or.v4i32(<4 x i32> [[VEC_PHI6]])
; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP9:%.*]] = freeze i1 [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = freeze i1 true
; CHECK-NEXT: [[RDX_SELECT8:%.*]] = select i1 [[TMP9]], i32 0, i32 0
; CHECK-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC4]]
; CHECK-NEXT: br i1 [[CMP_N9]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
>From 909c126dbb6d7f344bf78527b43132bccdb1854e Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 8 Apr 2026 18:15:35 +0100
Subject: [PATCH 3/5] !fixup address comments, thanks
---
.../Vectorize/LoopVectorizationPlanner.h | 21 +++++--
.../Transforms/Vectorize/LoopVectorize.cpp | 61 ++++++++-----------
.../Transforms/Vectorize/VPlanTransforms.cpp | 10 +--
3 files changed, 44 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 0c5744d8ff7a3..ce567843d57c7 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -272,6 +272,19 @@ class VPBuilder {
VPIRFlags(Pred, FastMathFlags()), {}, DL, Name));
}
+ /// Create an AnyOf reduction pattern: or-reduce \p ChainOp, freeze the
+ /// result, then select between \p TrueVal and \p FalseVal.
+ VPInstruction *createAnyOfReduction(VPValue *ChainOp, VPValue *TrueVal,
+ VPValue *FalseVal,
+ DebugLoc DL = DebugLoc::getUnknown()) {
+ VPIRFlags Flags(RecurKind::Or, /*IsOrdered=*/false, /*IsInLoop=*/false,
+ FastMathFlags());
+ auto *OrReduce = createNaryOp(VPInstruction::ComputeReductionResult,
+ {ChainOp}, Flags, DL);
+ auto *Freeze = createNaryOp(Instruction::Freeze, {OrReduce});
+ return createSelect(Freeze, TrueVal, FalseVal, DL, "rdx.select");
+ }
+
VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
@@ -674,10 +687,10 @@ class LoopVectorizationPlanner {
/// legal to vectorize the loop. This method creates VPlans using VPRecipes.
void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF);
- /// Add recipes to compute the final reduction result (using
- /// ComputeReductionResult) in the middle block. Selects are introduced for
- /// reductions between the phi and users outside the vector region when
- /// folding the tail.
+ /// Add ComputeReductionResult recipes to the middle block to compute the
+ /// final reduction results. Add Select recipes to the latch block when
+ /// folding tail, to feed ComputeReductionResult with the last or penultimate
+ /// iteration values according to the header mask.
void addReductionResultComputation(VPlanPtr &Plan,
VPRecipeBuilder &RecipeBuilder,
ElementCount MinVF);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 824a4b8d5903d..2f4705fa89778 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8328,7 +8328,6 @@ void LoopVectorizationPlanner::addReductionResultComputation(
NewExitingVPV =
Builder.createSelect(Cond, OrigExitingVPV, PhiR, {}, "", *PhiR);
OrigExitingVPV->replaceUsesWithIf(NewExitingVPV, [](VPUser &U, unsigned) {
- using namespace VPlanPatternMatch;
return match(&U,
m_VPInstruction<VPInstruction::ComputeReductionResult>());
});
@@ -8367,14 +8366,8 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPValue *NewVal = AnyOfSelect->getOperand(1) == PhiR
? AnyOfSelect->getOperand(2)
: AnyOfSelect->getOperand(1);
- VPIRFlags Flags(RecurKind::Or, /*IsOrdered=*/false, /*IsInLoop=*/false,
- FastMathFlags());
- auto *OrReduce =
- Builder.createNaryOp(VPInstruction::ComputeReductionResult,
- {NewExitingVPV}, Flags, ExitDL);
- auto *Freeze = Builder.createNaryOp(Instruction::Freeze, {OrReduce});
FinalReductionResult =
- Builder.createSelect(Freeze, NewVal, Start, {}, "rdx.select");
+ Builder.createAnyOfReduction(NewExitingVPV, NewVal, Start, ExitDL);
} else {
VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
PhiR->getFastMathFlags());
@@ -9007,7 +9000,6 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
ResumeV = cast<PHINode>(ReductionPhi->getUnderlyingInstr())
->getIncomingValueForBlock(L->getLoopPreheader());
- RecurKind RK = ReductionPhi->getRecurrenceKind();
// Check for FindIV pattern by looking for icmp user of RdxResult.
// The pattern is: select(icmp ne RdxResult, Sentinel), RdxResult, Start
@@ -9019,37 +9011,34 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
m_VPValue(SentinelVPV)));
});
- if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
+ RecurKind RK = ReductionPhi->getRecurrenceKind();
+ if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) || IsFindIV) {
Value *StartV = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
EPI.MainLoopIterationCountCheck);
- // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
- // start value; compare the final value from the main vector loop
- // to the start value.
- BasicBlock *PBB = cast<Instruction>(ResumeV)->getParent();
- IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt());
- ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
- if (auto *I = dyn_cast<Instruction>(ResumeV))
- InstsToMove.push_back(I);
- } else if (IsFindIV) {
- assert(SentinelVPV && "expected to find icmp using RdxResult");
-
- // Get the frozen start value from the main loop.
- Value *FrozenStartV = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
- EPI.MainLoopIterationCountCheck);
- if (auto *FreezeI = dyn_cast<FreezeInst>(FrozenStartV))
- ToFrozen[FreezeI->getOperand(0)] = FrozenStartV;
-
- // Adjust resume: select(icmp eq ResumeV, FrozenStartV), Sentinel,
- // ResumeV
BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
- Value *Cmp = Builder.CreateICmpEQ(ResumeV, FrozenStartV);
- if (auto *I = dyn_cast<Instruction>(Cmp))
- InstsToMove.push_back(I);
- ResumeV =
- Builder.CreateSelect(Cmp, SentinelVPV->getLiveInIRValue(), ResumeV);
- if (auto *I = dyn_cast<Instruction>(ResumeV))
- InstsToMove.push_back(I);
+
+ if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
+ // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
+ // start value; compare the final value from the main vector loop
+ // to the start value.
+ ResumeV = Builder.CreateICmpNE(ResumeV, StartV);
+ if (auto *I = dyn_cast<Instruction>(ResumeV))
+ InstsToMove.push_back(I);
+ } else {
+ assert(SentinelVPV && "expected to find icmp using RdxResult");
+ if (auto *FreezeI = dyn_cast<FreezeInst>(StartV))
+ ToFrozen[FreezeI->getOperand(0)] = StartV;
+
+ // Adjust resume: select(icmp eq ResumeV, StartV), Sentinel, ResumeV
+ Value *Cmp = Builder.CreateICmpEQ(ResumeV, StartV);
+ if (auto *I = dyn_cast<Instruction>(Cmp))
+ InstsToMove.push_back(I);
+ ResumeV = Builder.CreateSelect(Cmp, SentinelVPV->getLiveInIRValue(),
+ ResumeV);
+ if (auto *I = dyn_cast<Instruction>(ResumeV))
+ InstsToMove.push_back(I);
+ }
} else {
VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 910a35583844b..c6630ea5c3c85 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -5934,14 +5934,8 @@ void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
VPValue *OrVal = LoopBuilder.createOr(AnyOfPhi, AnyOfCond);
AnyOfPhi->setOperand(1, OrVal);
- VPIRFlags OrFlags(RecurKind::Or, /*IsOrdered=*/false,
- /*IsInLoop=*/false, FastMathFlags());
- auto *OrReduce = MiddleBuilder.createNaryOp(
- VPInstruction::ComputeReductionResult, {OrVal}, OrFlags, ExitDL);
- auto *Freeze =
- MiddleBuilder.createNaryOp(Instruction::Freeze, {OrReduce});
- NewRdxResult = MiddleBuilder.createSelect(
- Freeze, VectorRegionExitingVal, StartVPV, ExitDL);
+ NewRdxResult = MiddleBuilder.createAnyOfReduction(
+ OrVal, VectorRegionExitingVal, StartVPV, ExitDL);
// Initialize the IV reduction phi with the neutral element, not the
// original start value, to ensure correct min/max reduction results.
>From 6626064241553a3c810583819454c579193000d1 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 8 Apr 2026 22:25:37 +0100
Subject: [PATCH 4/5] !fixup address latest comments, thanks
---
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 3 ---
llvm/lib/Transforms/Vectorize/VPlan.h | 2 ++
llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 5 -----
llvm/test/Transforms/LoopVectorize/X86/cost-model.ll | 3 ++-
.../LoopVectorize/X86/pr109581-unused-blend.ll | 8 +++++---
.../epilog-vectorization-any-of-reductions.ll | 12 +++++++-----
6 files changed, 16 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d4f402d2911eb..899d449e87efd 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8358,8 +8358,6 @@ void LoopVectorizationPlanner::addReductionResultComputation(
AnyOfSelect = cast<VPRecipeBase>(*find_if(PhiR->users(), [](VPUser *U) {
return match(U, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
}));
- }
- if (AnyOfSelect) {
VPValue *Start = PhiR->getStartValue();
// NewVal is the non-phi operand of the select.
VPValue *NewVal = AnyOfSelect->getOperand(1) == PhiR
@@ -8994,7 +8992,6 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
auto IsReductionResult = [](VPRecipeBase *R) {
auto *VPI = dyn_cast<VPInstruction>(R);
return VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult;
- });
};
auto *RdxResult = cast<VPInstruction>(
vputils::findRecipe(ReductionPhi->getBackedgeValue(), IsReductionResult));
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index c150a283b2519..c75bc9dfdd235 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1267,6 +1267,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// scalars extracted from a vector, to be replaced by VF ExtractElement
/// VPInstructions.
Unpack,
+ /// Reduce the operands to the final reduction result using the operation
+ /// specified via the operations VPIRFlags.
ComputeReductionResult,
// Extracts the last part of its operand. Removed during unrolling.
ExtractLastPart,
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index ef5fef7a75fe0..b6ea7365a9a8b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -434,11 +434,6 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
R.addOperand(getValueForPart(Op1, Part));
continue;
}
- if (match(&R,
- m_ComputeAnyOfResult(m_VPValue(), m_VPValue(), m_VPValue()))) {
- addUniformForAllParts(cast<VPInstruction>(&R));
- continue;
- }
VPValue *Op0;
if (match(&R, m_ExtractLane(m_VPValue(Op0), m_VPValue(Op1)))) {
addUniformForAllParts(cast<VPInstruction>(&R));
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index eab7b9ba76a8c..60a3ea71bf06f 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -459,6 +459,7 @@ define i1 @any_of_cost(ptr %start, ptr %end) #0 {
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <2 x i1> [[TMP27]], [[TMP26]]
; CHECK-NEXT: [[TMP29:%.*]] = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> [[BIN_RDX]])
+; CHECK-NEXT: [[TMP30:%.*]] = freeze i1 [[TMP29]]
; CHECK-NEXT: br label %[[SCALAR_PH]]
; CHECK: [[SCALAR_PH]]:
;
@@ -471,7 +472,7 @@ loop:
%gep = getelementptr i8, ptr %ptr.iv, i64 8
%l = load ptr, ptr %gep, align 8
%cmp13.not.not = icmp eq ptr %l, null
- %any.of.next = select i1 %cmp13.not.not, i1 %any.of, i1 false
+ %any.of.next = select i1 %cmp13.not.not, i1 %any.of, i1 true
%ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 40
%cmp.not = icmp eq ptr %ptr.iv, %end
br i1 %cmp.not, label %exit, label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
index abbbbe0f1ad01..fd110335d68ea 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
@@ -71,11 +71,13 @@ define i32 @unused_blend_after_unrolling(ptr %p, i32 %a, i1 %c.1, i16 %x, i16 %y
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP25]], [[TMP24]]
; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX]])
+; CHECK-NEXT: [[TMP15:%.*]] = freeze i1 [[TMP27]]
+; CHECK-NEXT: [[RDX_SELECT:%.*]] = select i1 [[TMP15]], i32 1, i32 0
; CHECK-NEXT: br label %[[SCALAR_PH:.*]]
; CHECK: [[SCALAR_PH]]:
; CHECK-NEXT: br label %[[LOOP_HEADER:.*]]
; CHECK: [[LOOP_HEADER]]:
-; CHECK-NEXT: [[B:%.*]] = phi i32 [ 0, %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT: [[B:%.*]] = phi i32 [ [[RDX_SELECT]], %[[SCALAR_PH]] ], [ [[SEL:%.*]], %[[LOOP_LATCH:.*]] ]
; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 97, %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH]] ]
; CHECK-NEXT: br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[THEN:.*]]
; CHECK: [[THEN]]:
@@ -83,7 +85,7 @@ define i32 @unused_blend_after_unrolling(ptr %p, i32 %a, i1 %c.1, i16 %x, i16 %y
; CHECK-NEXT: br label %[[LOOP_LATCH]]
; CHECK: [[LOOP_LATCH]]:
; CHECK-NEXT: [[BLEND:%.*]] = phi i16 [ [[DIV]], %[[THEN]] ], [ 0, %[[LOOP_HEADER]] ]
-; CHECK-NEXT: [[SEL]] = select i1 [[C]], i32 [[B]], i32 0
+; CHECK-NEXT: [[SEL]] = select i1 [[C]], i32 [[B]], i32 1
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 100
; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
@@ -108,7 +110,7 @@ then:
loop.latch:
%blend = phi i16 [ %div, %then ], [ 0, %loop.header ]
- %sel = select i1 %c, i32 %b, i32 0
+ %sel = select i1 %c, i32 %b, i32 1
%iv.next = add i32 %iv, 1
%ec = icmp eq i32 %iv, 100
br i1 %ec, label %exit, label %loop.header
diff --git a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
index 8fea4fdb7d98c..c180bd9445a96 100644
--- a/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/epilog-vectorization-any-of-reductions.ll
@@ -220,6 +220,7 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
; CHECK-NEXT: br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]])
+; CHECK-NEXT: [[TMP6:%.*]] = freeze i1 [[TMP5]]
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
@@ -227,7 +228,7 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF3]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ false, [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i1 [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; CHECK-NEXT: [[TMP7:%.*]] = icmp ne i1 [[BC_MERGE_RDX]], false
; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[TMP0]], 4
@@ -253,11 +254,12 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
; CHECK-NEXT: br i1 [[TMP11]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: vec.epilog.middle.block:
; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]])
+; CHECK-NEXT: [[TMP13:%.*]] = freeze i1 [[TMP12]]
; CHECK-NEXT: [[CMP_N8:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC3]]
; CHECK-NEXT: br i1 [[CMP_N8]], label [[EXIT]], label [[VEC_EPILOG_SCALAR_PH]]
; CHECK: vec.epilog.scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi i64 [ [[N_VEC3]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ]
-; CHECK-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i1 [ false, [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ false, [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[ITER_CHECK]] ]
+; CHECK-NEXT: [[BC_MERGE_RDX17:%.*]] = phi i1 [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP6]], [[VEC_EPILOG_ITER_CHECK]] ], [ false, [[ITER_CHECK]] ]
; CHECK-NEXT: [[BC_RESUME_VAL7:%.*]] = phi i32 [ [[IND_END5]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[IND_END]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK]] ]
; CHECK-NEXT: br label [[LOOP:%.*]]
; CHECK: loop:
@@ -265,13 +267,13 @@ define i1 @any_of_reduction_i1_epilog(i64 %N, i32 %a) {
; CHECK-NEXT: [[RED_I1:%.*]] = phi i1 [ [[BC_MERGE_RDX17]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[SEL:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[IV_2:%.*]] = phi i32 [ [[BC_RESUME_VAL7]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP]] ]
; CHECK-NEXT: [[CMP_1:%.*]] = icmp eq i32 [[IV_2]], [[A]]
-; CHECK-NEXT: [[SEL]] = select i1 [[CMP_1]], i1 [[RED_I1]], i1 false
+; CHECK-NEXT: [[SEL]] = select i1 [[CMP_1]], i1 [[RED_I1]], i1 true
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[IV_2_NEXT]] = add i32 [[IV_2]], 1
; CHECK-NEXT: [[CMP_2:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[CMP_2]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK: exit:
-; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i1 [ [[SEL]], [[LOOP]] ], [ false, [[MIDDLE_BLOCK]] ], [ false, [[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[SEL_LCSSA:%.*]] = phi i1 [ [[SEL]], [[LOOP]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ], [ [[TMP13]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i1 [[SEL_LCSSA]]
;
entry:
@@ -282,7 +284,7 @@ loop:
%red.i1 = phi i1 [ false, %entry ], [ %sel, %loop ]
%iv.2 = phi i32 [ 0, %entry ], [ %iv.2.next, %loop ]
%cmp.1 = icmp eq i32 %iv.2, %a
- %sel = select i1 %cmp.1, i1 %red.i1, i1 false
+ %sel = select i1 %cmp.1, i1 %red.i1, i1 true
%iv.next = add i64 %iv, 1
%iv.2.next = add i32 %iv.2, 1
%cmp.2 = icmp eq i64 %iv, %N
>From a3c3ce5511e53ef4a063e2bfa600e9cbbbd4bc81 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 10 Apr 2026 13:41:25 +0100
Subject: [PATCH 5/5] !fixup assert chainop bool
---
.../Vectorize/LoopVectorizationPlanner.h | 9 +-
.../Transforms/Vectorize/LoopVectorize.cpp | 86 +++++++++++--------
llvm/lib/Transforms/Vectorize/VPlan.cpp | 15 ++++
llvm/lib/Transforms/Vectorize/VPlan.h | 2 +-
4 files changed, 65 insertions(+), 47 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index ce567843d57c7..602571df5e6db 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -276,14 +276,7 @@ class VPBuilder {
/// result, then select between \p TrueVal and \p FalseVal.
VPInstruction *createAnyOfReduction(VPValue *ChainOp, VPValue *TrueVal,
VPValue *FalseVal,
- DebugLoc DL = DebugLoc::getUnknown()) {
- VPIRFlags Flags(RecurKind::Or, /*IsOrdered=*/false, /*IsInLoop=*/false,
- FastMathFlags());
- auto *OrReduce = createNaryOp(VPInstruction::ComputeReductionResult,
- {ChainOp}, Flags, DL);
- auto *Freeze = createNaryOp(Instruction::Freeze, {OrReduce});
- return createSelect(Freeze, TrueVal, FalseVal, DL, "rdx.select");
- }
+ DebugLoc DL = DebugLoc::getUnknown());
VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,
DebugLoc DL = DebugLoc::getUnknown(),
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 63343055d6d8e..878630d1fd8f3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8384,18 +8384,53 @@ void LoopVectorizationPlanner::addReductionResultComputation(
VPInstruction *FinalReductionResult;
VPBuilder::InsertPointGuard Guard(Builder);
Builder.setInsertPoint(MiddleVPBB, IP);
- // For AnyOf reductions, find the select among PhiR's users. This is used
- // both to find NewVal for the final result and to adjust the reduction.
- VPRecipeBase *AnyOfSelect = nullptr;
+ // For AnyOf reductions, find the select among PhiR's users and convert
+ // the reduction phi to operate on bools before creating the final
+ // reduction result.
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RecurrenceKind)) {
- AnyOfSelect = cast<VPRecipeBase>(*find_if(PhiR->users(), [](VPUser *U) {
- return match(U, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
- }));
+ auto *AnyOfSelect =
+ cast<VPSingleDefRecipe>(*find_if(PhiR->users(), [](VPUser *U) {
+ return match(U, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
+ }));
VPValue *Start = PhiR->getStartValue();
+ bool TrueValIsPhi = AnyOfSelect->getOperand(1) == PhiR;
// NewVal is the non-phi operand of the select.
- VPValue *NewVal = AnyOfSelect->getOperand(1) == PhiR
- ? AnyOfSelect->getOperand(2)
- : AnyOfSelect->getOperand(1);
+ VPValue *NewVal = TrueValIsPhi ? AnyOfSelect->getOperand(2)
+ : AnyOfSelect->getOperand(1);
+
+ // Adjust AnyOf reductions; replace the reduction phi for the selected
+ // value with a boolean reduction phi node to check if the condition is
+ // true in any iteration. The final value is selected by the final
+ // ComputeReductionResult.
+ VPValue *Cmp = AnyOfSelect->getOperand(0);
+ // If the compare is checking the reduction PHI node, adjust it to check
+ // the start value.
+ if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
+ CmpR->replaceUsesOfWith(PhiR, PhiR->getStartValue());
+ Builder.setInsertPoint(AnyOfSelect);
+
+ // If the true value of the select is the reduction phi, the new value
+ // is selected if the negated condition is true in any iteration.
+ if (TrueValIsPhi)
+ Cmp = Builder.createNot(Cmp);
+ VPValue *Or = Builder.createOr(PhiR, Cmp);
+ // Only replace uses inside the vector region with Or. External uses
+ // (e.g. scalar preheader resume phis) must be replaced by the user
+ // update loop below with FinalReductionResult.
+ AnyOfSelect->replaceUsesWithIf(Or, [](VPUser &U, unsigned) {
+ return cast<VPRecipeBase>(&U)->getRegion();
+ });
+ ToDelete.push_back(AnyOfSelect);
+
+ // Convert the reduction phi to operate on bools.
+ PhiR->setOperand(0, Plan->getFalse());
+
+ // Update NewExitingVPV if it was pointing to the now-replaced select.
+ if (NewExitingVPV == AnyOfSelect)
+ NewExitingVPV = Or;
+
+ Builder.setInsertPoint(MiddleVPBB, IP);
+
FinalReductionResult =
Builder.createAnyOfReduction(NewExitingVPV, NewVal, Start, ExitDL);
} else {
@@ -8460,32 +8495,6 @@ void LoopVectorizationPlanner::addReductionResultComputation(
cast<VPInstruction>(U)->replaceAllUsesWith(FinalReductionResult);
}
- // Adjust AnyOf reductions; replace the reduction phi for the selected value
- // with a boolean reduction phi node to check if the condition is true in
- // any iteration. The final value is selected by the final
- // ComputeReductionResult.
- if (AnyOfSelect) {
- VPValue *Cmp = AnyOfSelect->getOperand(0);
- // If the compare is checking the reduction PHI node, adjust it to check
- // the start value.
- if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
- CmpR->replaceUsesOfWith(PhiR, PhiR->getStartValue());
- Builder.setInsertPoint(AnyOfSelect);
-
- // If the true value of the select is the reduction phi, the new value is
- // selected if the negated condition is true in any iteration.
- if (AnyOfSelect->getOperand(1) == PhiR)
- Cmp = Builder.createNot(Cmp);
- VPValue *Or = Builder.createOr(PhiR, Cmp);
- AnyOfSelect->getVPSingleValue()->replaceAllUsesWith(Or);
- // Delete AnyOfSelect now that it has invalid types.
- ToDelete.push_back(AnyOfSelect);
-
- // Convert the reduction phi to operate on bools.
- PhiR->setOperand(0, Plan->getFalse());
- continue;
- }
-
RecurKind RK = PhiR->getRecurrenceKind();
if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) &&
!RecurrenceDescriptor::isFindIVRecurrenceKind(RK) &&
@@ -9045,10 +9054,11 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
RecurKind RK = ReductionPhi->getRecurrenceKind();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK) || IsFindIV) {
- Value *StartV = cast<PHINode>(ResumeV)->getIncomingValueForBlock(
+ auto *ResumePhi = cast<PHINode>(ResumeV);
+ Value *StartV = ResumePhi->getIncomingValueForBlock(
EPI.MainLoopIterationCountCheck);
- BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
- IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
+ IRBuilder<> Builder(ResumePhi->getParent(),
+ ResumePhi->getParent()->getFirstNonPHIIt());
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK)) {
// VPReductionPHIRecipes for AnyOf reductions expect a boolean as
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 3ba5ac582fa9a..ed730894be107 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1613,6 +1613,21 @@ std::string VPSlotTracker::getOrCreateName(const VPValue *V) const {
return "<badref>";
}
+VPInstruction *VPBuilder::createAnyOfReduction(VPValue *ChainOp,
+ VPValue *TrueVal,
+ VPValue *FalseVal, DebugLoc DL) {
+ assert(VPTypeAnalysis(*getInsertBlock()->getPlan())
+ .inferScalarType(ChainOp)
+ ->isIntegerTy(1) &&
+ "ChainOp must be i1 for AnyOf reduction");
+ VPIRFlags Flags(RecurKind::Or, /*IsOrdered=*/false, /*IsInLoop=*/false,
+ FastMathFlags());
+ auto *OrReduce =
+ createNaryOp(VPInstruction::ComputeReductionResult, {ChainOp}, Flags, DL);
+ auto *Freeze = createNaryOp(Instruction::Freeze, {OrReduce}, DL);
+ return createSelect(Freeze, TrueVal, FalseVal, DL, "rdx.select");
+}
+
bool LoopVectorizationPlanner::getDecisionAndClampRange(
const std::function<bool(ElementCount)> &Predicate, VFRange &Range) {
assert(!Range.isEmpty() && "Trying to test an empty VF range.");
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 55fd1f9bd8abc..41d3037bea0d3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1268,7 +1268,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags,
/// VPInstructions.
Unpack,
/// Reduce the operands to the final reduction result using the operation
- /// specified via the operations VPIRFlags.
+ /// specified via the operation's VPIRFlags.
ComputeReductionResult,
// Extracts the last part of its operand. Removed during unrolling.
ExtractLastPart,
More information about the llvm-commits
mailing list