[llvm] [LV] Handle partial reductions with predication (PR #194859)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 04:05:51 PDT 2026
https://github.com/jacob-crawley updated https://github.com/llvm/llvm-project/pull/194859
>From 7084931ff6270b8ac09d3f40a80fb2e184df8df3 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 24 Apr 2026 14:49:09 +0000
Subject: [PATCH 01/15] [LV] Handle partial reductions with predication
Enables partial reductions to be used when a reduction is
gated by a condition inside a loop.
Previously when a loop contained a condition, this would introduce a
BLEND as the backedge of the phi, which prevented a partial
reduction from being used in a pattern where it would normally be
created.
This patch checks for a BLEND in the VPlan to determine whether
one of its operands can be transformed into a partial reduction.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 49 +
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 4 +-
.../AArch64/partial-reduce-with-predicate.ll | 920 ++++++++++++++++++
3 files changed, 972 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bdffe0c245a34..37041f236d9f2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6548,6 +6548,29 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
VPValue *ExitValue = cast_or_null<VPInstruction>(
findUserOf(WidenRecipe, m_Select(m_VPValue(Cond), m_Specific(WidenRecipe),
m_Specific(RdxPhi))));
+
+ // Look for a VPBlendRecipe user of the WidenRecipe. When the reduction is
+ // gated by an `if` in the loop body, the blend merges the updated value
+ // (WidenRecipe) and the unchanged phi value (RdxPhi).
+ if (!ExitValue) {
+ for (VPUser *U : WidenRecipe->users()) {
+ auto *Blend = dyn_cast<VPBlendRecipe>(cast<VPRecipeBase>(U));
+ if (!Blend || Blend->getNumIncomingValues() != 2)
+ continue;
+ VPValue *V0 = Blend->getIncomingValue(0);
+ VPValue *V1 = Blend->getIncomingValue(1);
+
+ if ((V0 == WidenRecipe) && (V1 == RdxPhi) ||
+ (V1 == WidenRecipe) && (V0 == RdxPhi)) {
+ // Use the mask of the WidenRecipe incoming as the reduction condition.
+ unsigned maskIdx = (V0 == WidenRecipe) ? 0 : 1;
+ Cond = Blend->getMask(maskIdx);
+ ExitValue = Blend;
+ break;
+ }
+ }
+ }
+
bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
RdxPhi->getBackedgeValue() == ExitValue;
assert((!ExitValue || IsLastInChain) &&
@@ -6778,6 +6801,19 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
VPValue *ExitValue = RdxResult->getOperand(0);
match(ExitValue, m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
+ // Check for a reduction in the operand of a Blend.
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(ExitValue)) {
+ if(Blend->getNumIncomingValues() == 2) {
+ VPValue *V0 = Blend->getIncomingValue(0);
+ VPValue *V1 = Blend->getIncomingValue(1);
+
+ if (V0 == RedPhiR)
+ ExitValue = V1;
+ if (V1 == RedPhiR)
+ ExitValue = V0;
+ }
+ }
+
SmallVector<VPPartialReductionChain> Chain;
RecurKind RK = RedPhiR->getRecurrenceKind();
Type *PhiType = RedPhiR->getScalarType();
@@ -6906,6 +6942,19 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
return PhiR == RedPhiR;
+
+ // Predicated reductions introduced by an 'if' inside the loop
+ // will be represented by a VPBlendRecipe with two incoming values,
+ // the reduction PHI and the reduction Bin Op.
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(U)) {
+ if (Blend->getNumIncomingValues() != 2)
+ return false;
+
+ VPValue *V0 = Blend->getIncomingValue(0);
+ VPValue *V1 = Blend->getIncomingValue(1);
+ return (V0 == Chain.ReductionBinOp && V1 == RedPhiR) || (V1 == Chain.ReductionBinOp && V0 == RedPhiR);
+ }
+
auto *R = cast<VPSingleDefRecipe>(U);
return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
match(R, m_ComputeReductionResult(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index f477faea58d1a..a80b9e1326798 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -848,9 +848,11 @@ VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
findUserOf<VPInstruction::ComputeReductionResult>(BackedgeVal))
return Res;
- // Look through selects inserted for tail folding or predicated reductions.
+ // Look through selects inserted for tail folding or predicated reductions.
VPRecipeBase *SelR =
findUserOf(BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
+ if (!SelR)
+ SelR = dyn_cast_or_null<VPBlendRecipe>(BackedgeVal->getDefiningRecipe());
if (!SelR)
return nullptr;
return findUserOf<VPInstruction::ComputeReductionResult>(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
new file mode 100644
index 0000000000000..7d53745b693d4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -0,0 +1,920 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVE1
+; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVED
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-MAXBW
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @pred_reduction(
+; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT: [[FOR_BODY_PREHEADER:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-INTERLEAVE1: [[FOR_COND_CLEANUP]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM_08:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDVARS_IV]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP11]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[TMP12]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_08]], [[CONV]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_08]], %[[FOR_COND_CLEANUP]] ]
+; CHECK-INTERLEAVE1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVE1: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction(
+; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT: [[FOR_BODY_PREHEADER:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP11]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP12]])
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP13]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP14]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-INTERLEAVED: [[FOR_COND_CLEANUP]]:
+; CHECK-INTERLEAVED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM_08:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDVARS_IV]]
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP17]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[TMP18]] to i32
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_08]], [[CONV]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_08]], %[[FOR_COND_CLEANUP]] ]
+; CHECK-INTERLEAVED-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVED: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @pred_reduction(
+; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-MAXBW-NEXT: [[FOR_BODY_PREHEADER:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-MAXBW: [[FOR_COND_CLEANUP]]:
+; CHECK-MAXBW-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM_08:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDVARS_IV]]
+; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP11]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[CONV:%.*]] = zext i8 [[TMP12]] to i32
+; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_08]], [[CONV]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_08]], %[[FOR_COND_CLEANUP]] ]
+; CHECK-MAXBW-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-MAXBW: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @pred_reduction_sext(
+; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = sext i8 [[VAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction_sext(
+; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP11]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP12]])
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP13]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP14]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = sext i8 [[VAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @pred_reduction_sext(
+; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-MAXBW: [[FOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[CONV:%.*]] = sext i8 [[VAL]] to i32
+; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-MAXBW: [[EXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = sext i8 %val to i32
+ %add = add nsw i32 %sum, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VA:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VB:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV_A:%.*]] = zext i8 [[VA]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[CONV_B:%.*]] = zext i8 [[VB]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV_A]], [[CONV_B]]
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP11]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP15]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP16]])
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP17]], [[TMP18]]
+; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP19]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP20]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VA:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VB:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV_A:%.*]] = zext i8 [[VA]] to i32
+; CHECK-INTERLEAVED-NEXT: [[CONV_B:%.*]] = zext i8 [[VB]] to i32
+; CHECK-INTERLEAVED-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV_A]], [[CONV_B]]
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP22]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-MAXBW: [[FOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[VA:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[VB:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-MAXBW-NEXT: [[CONV_A:%.*]] = zext i8 [[VA]] to i32
+; CHECK-MAXBW-NEXT: [[CONV_B:%.*]] = zext i8 [[VB]] to i32
+; CHECK-MAXBW-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV_A]], [[CONV_B]]
+; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-MAXBW: [[EXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %arrayidx2, align 1
+ %arrayidx4 = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %arrayidx4, align 1
+ %ext.a = zext i8 %load.a to i32
+ %ext.b = zext i8 %load.b to i32
+ %mul = mul nuw nsw i32 %ext.a, %ext.b
+ %add = add nsw i32 %sum, %mul
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @pred_sub_reduction(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP14:%.*]] = sub i32 0, [[TMP13]]
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
+; CHECK-INTERLEAVE1-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM]], [[MUL]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[SUB]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @pred_sub_reduction(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP11]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP15]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP16]])
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP17]], [[TMP18]]
+; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP19]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP20]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = sub i32 0, [[TMP22]]
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP23]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVED-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
+; CHECK-INTERLEAVED-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
+; CHECK-INTERLEAVED-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM]], [[MUL]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[SUB]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP23]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @pred_sub_reduction(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT: [[TMP14:%.*]] = sub i32 0, [[TMP13]]
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-MAXBW: [[FOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-MAXBW-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-MAXBW-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
+; CHECK-MAXBW-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
+; CHECK-MAXBW-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM]], [[MUL]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[SUB]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-MAXBW: [[EXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %arrayidx2, align 1
+ %arrayidx4 = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %arrayidx4, align 1
+ %ext.a = zext i8 %load.a to i32
+ %ext.b = zext i8 %load.b to i32
+ %mul = mul nuw nsw i32 %ext.a, %ext.b
+ %sub = sub nsw i32 %sum, %mul
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %sub, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
>From dbb2634bfa5e011d125a6379dbeb12555d60c582 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 17 Jun 2026 10:45:27 +0000
Subject: [PATCH 02/15] Use m_SelectLike matcher to check for blend recipes
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 56 ++++++-------------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 12 ++--
.../AArch64/partial-reduce-with-predicate.ll | 36 ++++--------
3 files changed, 37 insertions(+), 67 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 37041f236d9f2..ea3390b622c83 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6543,33 +6543,21 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
}
// Check if WidenRecipe is the final result of the reduction. If so look
- // through selects for predicated reductions.
+ // through select-like recipes for predicated reductions.
VPValue *Cond = nullptr;
- VPValue *ExitValue = cast_or_null<VPInstruction>(
- findUserOf(WidenRecipe, m_Select(m_VPValue(Cond), m_Specific(WidenRecipe),
- m_Specific(RdxPhi))));
-
- // Look for a VPBlendRecipe user of the WidenRecipe. When the reduction is
- // gated by an `if` in the loop body, the blend merges the updated value
- // (WidenRecipe) and the unchanged phi value (RdxPhi).
- if (!ExitValue) {
- for (VPUser *U : WidenRecipe->users()) {
- auto *Blend = dyn_cast<VPBlendRecipe>(cast<VPRecipeBase>(U));
- if (!Blend || Blend->getNumIncomingValues() != 2)
- continue;
- VPValue *V0 = Blend->getIncomingValue(0);
- VPValue *V1 = Blend->getIncomingValue(1);
-
- if ((V0 == WidenRecipe) && (V1 == RdxPhi) ||
- (V1 == WidenRecipe) && (V0 == RdxPhi)) {
- // Use the mask of the WidenRecipe incoming as the reduction condition.
- unsigned maskIdx = (V0 == WidenRecipe) ? 0 : 1;
- Cond = Blend->getMask(maskIdx);
- ExitValue = Blend;
- break;
- }
+ VPRecipeBase *ExitRecipe = findUserOf(
+ WidenRecipe, m_SelectLike(m_VPValue(Cond), m_Specific(WidenRecipe),
+ m_Specific(RdxPhi)));
+ if (!ExitRecipe) {
+ ExitRecipe = findUserOf(WidenRecipe,
+ m_SelectLike(m_VPValue(Cond), m_Specific(RdxPhi),
+ m_Specific(WidenRecipe)));
+ if (ExitRecipe) {
+ VPBuilder Builder(WidenRecipe);
+ Cond = Builder.createNot(Cond, ExitRecipe->getDebugLoc());
}
}
+ VPValue *ExitValue = cast_or_null<VPSingleDefRecipe>(ExitRecipe);
bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
RdxPhi->getBackedgeValue() == ExitValue;
@@ -6799,20 +6787,12 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
if (!RdxResult)
return std::nullopt;
VPValue *ExitValue = RdxResult->getOperand(0);
- match(ExitValue, m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
-
- // Check for a reduction in the operand of a Blend.
- if (auto *Blend = dyn_cast<VPBlendRecipe>(ExitValue)) {
- if(Blend->getNumIncomingValues() == 2) {
- VPValue *V0 = Blend->getIncomingValue(0);
- VPValue *V1 = Blend->getIncomingValue(1);
-
- if (V0 == RedPhiR)
- ExitValue = V1;
- if (V1 == RedPhiR)
- ExitValue = V0;
- }
- }
+ VPValue *SelectedValue = nullptr;
+ if (match(ExitValue, m_SelectLike(m_VPValue(), m_VPValue(SelectedValue),
+ m_Specific(RedPhiR))) ||
+ match(ExitValue, m_SelectLike(m_VPValue(), m_Specific(RedPhiR),
+ m_VPValue(SelectedValue))))
+ ExitValue = SelectedValue;
SmallVector<VPPartialReductionChain> Chain;
RecurKind RK = RedPhiR->getRecurrenceKind();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index a80b9e1326798..ed88dca2e732c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -848,11 +848,13 @@ VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
findUserOf<VPInstruction::ComputeReductionResult>(BackedgeVal))
return Res;
- // Look through selects inserted for tail folding or predicated reductions.
- VPRecipeBase *SelR =
- findUserOf(BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
- if (!SelR)
- SelR = dyn_cast_or_null<VPBlendRecipe>(BackedgeVal->getDefiningRecipe());
+ // Look through select-like recipes inserted for tail folding or predicated
+ // reductions.
+ VPRecipeBase *SelR = findUserOf(
+ BackedgeVal, m_SelectLike(m_VPValue(), m_VPValue(), m_VPValue()));
+ if (!SelR &&
+ match(BackedgeVal, m_SelectLike(m_VPValue(), m_VPValue(), m_VPValue())))
+ SelR = BackedgeVal->getDefiningRecipe();
if (!SelR)
return nullptr;
return findUserOf<VPInstruction::ComputeReductionResult>(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index 7d53745b693d4..0af873aef91c5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -15,8 +15,7 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -72,8 +71,7 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
@@ -140,8 +138,7 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -226,8 +223,7 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -283,8 +279,7 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
@@ -351,8 +346,7 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -437,8 +431,7 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -502,8 +495,7 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
@@ -582,8 +574,7 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -680,8 +671,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -746,8 +736,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
@@ -827,8 +816,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
>From f48d0430e812330d60d788fd40a9154c96fa211b Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 17 Jun 2026 13:50:08 +0000
Subject: [PATCH 03/15] formatting
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ea3390b622c83..7eb01626a0d0b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6932,7 +6932,8 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
VPValue *V0 = Blend->getIncomingValue(0);
VPValue *V1 = Blend->getIncomingValue(1);
- return (V0 == Chain.ReductionBinOp && V1 == RedPhiR) || (V1 == Chain.ReductionBinOp && V0 == RedPhiR);
+ return (V0 == Chain.ReductionBinOp && V1 == RedPhiR) ||
+ (V1 == Chain.ReductionBinOp && V0 == RedPhiR);
}
auto *R = cast<VPSingleDefRecipe>(U);
>From ea7e7af9023b0c93c7e7d0310889a51706d7358a Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 19 Jun 2026 11:26:31 +0000
Subject: [PATCH 04/15] Add BlendMask for tail-folding and chained reductions
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 129 ++++++++++++++++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 10 +-
2 files changed, 120 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7eb01626a0d0b..2c598e1aab24c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6355,6 +6355,15 @@ struct VPPartialReductionChain {
/// is `1 - AccumulatorOpIdx`.
unsigned AccumulatorOpIdx;
unsigned ScaleFactor;
+
+ /// Mask from a VPBlendRecipe that guards the reduction (e.g. an `if` inside
+ /// the loop body).
+ VPValue *BlendMask = nullptr;
+
+ /// If true, negate the BlendMask before passing to the partial reduction.
+ /// This is needed for normalized blends where ReductionBinOp is the first
+ /// incoming value and selected when BlendMask is false.
+ bool NegateBlendMask = false;
};
static VPSingleDefRecipe *
@@ -6544,23 +6553,60 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// Check if WidenRecipe is the final result of the reduction. If so look
// through select-like recipes for predicated reductions.
- VPValue *Cond = nullptr;
+ //
+ // There are up to two sources of predication:
+ // 1) Chain.Mask: captured from a VPBlendRecipe that gates this link with
+ // an inner `if` mask. May need negation (see Chain.NegateMask).
+ // 2) A latch select: introduced by tail-folding and/or
+ // non-predicated reductions to select between the updated and previous
+ // reduction values.
+ // Both must be combined (logical-and) and absorbed into the partial
+ // reduction's Cond, and both the blend and the latch select must be
+ // replaced by the partial reduction.
+ VPValue *Cond = Chain.BlendMask;
+ if (Cond && Chain.NegateBlendMask) {
+ VPBuilder Builder(WidenRecipe);
+ Cond = Builder.createNot(Cond, WidenRecipe->getDebugLoc());
+ }
+
+ // If the chain saw a blend, the latch select uses the
+ // blend rather than the WidenRecipe directly.
+ VPValue *SearchVal = WidenRecipe;
+ VPBlendRecipe *BlendUser = nullptr;
+ if (Chain.BlendMask) {
+ BlendUser = findUserOf<VPBlendRecipe>(WidenRecipe);
+ if (BlendUser)
+ SearchVal = BlendUser;
+ }
+
+ VPValue *LatchCond = nullptr;
VPRecipeBase *ExitRecipe = findUserOf(
- WidenRecipe, m_SelectLike(m_VPValue(Cond), m_Specific(WidenRecipe),
- m_Specific(RdxPhi)));
+ SearchVal, m_SelectLike(m_VPValue(LatchCond), m_Specific(SearchVal),
+ m_Specific(RdxPhi)));
if (!ExitRecipe) {
- ExitRecipe = findUserOf(WidenRecipe,
- m_SelectLike(m_VPValue(Cond), m_Specific(RdxPhi),
- m_Specific(WidenRecipe)));
+ ExitRecipe = findUserOf(SearchVal, m_SelectLike(m_VPValue(LatchCond),
+ m_Specific(RdxPhi),
+ m_Specific(SearchVal)));
if (ExitRecipe) {
VPBuilder Builder(WidenRecipe);
- Cond = Builder.createNot(Cond, ExitRecipe->getDebugLoc());
+ LatchCond = Builder.createNot(LatchCond, ExitRecipe->getDebugLoc());
}
}
+
+ // Combine the blend mask and the latch select mask.
+ if (LatchCond && Cond) {
+ VPBuilder Builder(WidenRecipe);
+ Cond =
+ Builder.createLogicalAnd(LatchCond, Cond, WidenRecipe->getDebugLoc());
+ } else if (LatchCond) {
+ Cond = LatchCond;
+ }
+
VPValue *ExitValue = cast_or_null<VPSingleDefRecipe>(ExitRecipe);
bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
- RdxPhi->getBackedgeValue() == ExitValue;
+ RdxPhi->getBackedgeValue() == ExitValue ||
+ (BlendUser && RdxPhi->getBackedgeValue() == BlendUser);
assert((!ExitValue || IsLastInChain) &&
"if we found ExitValue, it must match RdxPhi's backedge value");
@@ -6575,8 +6621,12 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
RdxUnordered{/*VFScaleFactor=*/Chain.ScaleFactor});
PartialRed->insertBefore(WidenRecipe);
- if (Cond)
+ // Replace uses of the latch select (if found) with PartialRed; otherwise
+ // the blend directly produces the reduction backedge value.
+ if (ExitValue)
ExitValue->replaceAllUsesWith(PartialRed);
+ if (BlendUser)
+ BlendUser->replaceAllUsesWith(PartialRed);
WidenRecipe->replaceAllUsesWith(PartialRed);
// For cost-model purposes, fold this into a VPExpression.
@@ -6802,6 +6852,44 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
// Work backwards from the ExitValue examining each reduction operation.
VPValue *CurrentValue = ExitValue;
while (CurrentValue != RedPhiR) {
+ VPValue *LinkMask = nullptr;
+ bool LinkNegateMask = false;
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentValue)) {
+ // Look through an incoming blend. We handle both:
+ // * Normalized blends ([I0, I1, M1]): introduced by inner `if`.
+ // * Non-normalized blends ([I0, M0, I1, M1]): introduced by an inner
+ // `if` combined with tail-folding (M0 and M1 already include the
+ // header / lane mask).
+ if (Blend->getNumIncomingValues() != 2)
+ return std::nullopt;
+
+ VPValue *V0 = Blend->getIncomingValue(0);
+ VPValue *V1 = Blend->getIncomingValue(1);
+ auto IsBinOpWiden = [](VPValue *V) {
+ auto *W = dyn_cast_or_null<VPWidenRecipe>(V);
+ return W && Instruction::isBinaryOp(W->getOpcode());
+ };
+
+ // Identify which incoming is the BinOp.
+ if (IsBinOpWiden(V1)) {
+ // Both forms: V1 is selected when M1 is true.
+ CurrentValue = V1;
+ LinkMask = Blend->getMask(1);
+ } else if (IsBinOpWiden(V0)) {
+ CurrentValue = V0;
+ if (Blend->isNormalized()) {
+ // Normalized [I0, I1, M1]: V0 is default, fires when M1 is false.
+ LinkMask = Blend->getMask(1);
+ LinkNegateMask = true;
+ } else {
+ // Non-normalized [I0, M0, I1, M1]: V0 fires when M0 is true.
+ LinkMask = Blend->getMask(0);
+ }
+ } else {
+ return std::nullopt;
+ }
+ }
+
auto *UpdateR = dyn_cast<VPWidenRecipe>(CurrentValue);
if (!UpdateR || !Instruction::isBinaryOp(UpdateR->getOpcode()))
return std::nullopt;
@@ -6831,7 +6919,8 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
VPPartialReductionChain Link(
{UpdateR, *ExtendedOp, RK,
PrevValue == UpdateR->getOperand(0) ? 0U : 1U,
- static_cast<unsigned>(PHISize.getKnownScalarFactor(ExtSrcSize))});
+ static_cast<unsigned>(PHISize.getKnownScalarFactor(ExtSrcSize)),
+ LinkMask, LinkNegateMask});
Chain.push_back(Link);
CurrentValue = PrevValue;
}
@@ -6932,8 +7021,24 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
VPValue *V0 = Blend->getIncomingValue(0);
VPValue *V1 = Blend->getIncomingValue(1);
- return (V0 == Chain.ReductionBinOp && V1 == RedPhiR) ||
- (V1 == Chain.ReductionBinOp && V0 == RedPhiR);
+ if (!((V0 == Chain.ReductionBinOp && V1 == RedPhiR) ||
+ (V1 == Chain.ReductionBinOp && V0 == RedPhiR)))
+ return false;
+
+ // The blend's users must be the reduction PHI's backedge, the
+ // compute-reduction-result, the latch select introduced
+ // by tail-folding, or another scaled-reduction with the same
+ // scale factor for chained reductions.
+ return all_of(Blend->users(), [&](VPUser *BU) {
+ if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(BU))
+ return PhiR == RedPhiR;
+ auto *R = dyn_cast<VPSingleDefRecipe>(BU);
+ return R &&
+ (Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
+ match(R, m_ComputeReductionResult(m_Specific(Blend))) ||
+ match(R, m_Select(m_VPValue(), m_Specific(Blend),
+ m_Specific(RedPhiR))));
+ });
}
auto *R = cast<VPSingleDefRecipe>(U);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index ed88dca2e732c..f477faea58d1a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -848,13 +848,9 @@ VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
findUserOf<VPInstruction::ComputeReductionResult>(BackedgeVal))
return Res;
- // Look through select-like recipes inserted for tail folding or predicated
- // reductions.
- VPRecipeBase *SelR = findUserOf(
- BackedgeVal, m_SelectLike(m_VPValue(), m_VPValue(), m_VPValue()));
- if (!SelR &&
- match(BackedgeVal, m_SelectLike(m_VPValue(), m_VPValue(), m_VPValue())))
- SelR = BackedgeVal->getDefiningRecipe();
+ // Look through selects inserted for tail folding or predicated reductions.
+ VPRecipeBase *SelR =
+ findUserOf(BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
if (!SelR)
return nullptr;
return findUserOf<VPInstruction::ComputeReductionResult>(
>From cb3a700e2b2167c5fc664a971a92e74789f63c79 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 19 Jun 2026 13:44:30 +0000
Subject: [PATCH 05/15] added chained reduction test
---
.../AArch64/partial-reduce-with-predicate.ll | 241 ++++++++++++++++++
1 file changed, 241 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index 0af873aef91c5..e22f0e4899f45 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -905,4 +905,245 @@ exit:
ret i32 %sum.1
}
+define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @chained_pred_reduction(
+; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1:%.*]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @chained_pred_reduction(
+; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP12]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP13]])
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP14]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 16 x i8>, ptr [[TMP15]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD5]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP16]])
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD6]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE8]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE4]], <vscale x 16 x i32> [[TMP17]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
+; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1:%.*]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-INTERLEAVED-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @chained_pred_reduction(
+; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-MAXBW: [[FOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_1:%.*]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-MAXBW-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-MAXBW-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
+; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-MAXBW: [[EXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ ; Chain to result of blend:
+ %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv
+ %bval = load i8, ptr %b.gep, align 1
+ %bconv = zext i8 %bval to i32
+ %sum.2 = add nsw i32 %sum.1, %bconv
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.2
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
>From 66d73e42b86b9f7483679e299c54dd5e41fc0074 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Mon, 22 Jun 2026 16:18:39 +0000
Subject: [PATCH 06/15] Store Blend in PartialReductionChain instead of mask
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 165 ++++--------------
1 file changed, 34 insertions(+), 131 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2c598e1aab24c..7e992483d1933 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6356,14 +6356,8 @@ struct VPPartialReductionChain {
unsigned AccumulatorOpIdx;
unsigned ScaleFactor;
- /// Mask from a VPBlendRecipe that guards the reduction (e.g. an `if` inside
- /// the loop body).
- VPValue *BlendMask = nullptr;
-
- /// If true, negate the BlendMask before passing to the partial reduction.
- /// This is needed for normalized blends where ReductionBinOp is the first
- /// incoming value and selected when BlendMask is false.
- bool NegateBlendMask = false;
+ /// Optional blend guarding the reduction.
+ VPBlendRecipe *Blend = nullptr;
};
static VPSingleDefRecipe *
@@ -6552,61 +6546,28 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
}
// Check if WidenRecipe is the final result of the reduction. If so look
- // through select-like recipes for predicated reductions.
- //
- // There are up to two sources of predication:
- // 1) Chain.Mask: captured from a VPBlendRecipe that gates this link with
- // an inner `if` mask. May need negation (see Chain.NegateMask).
- // 2) A latch select: introduced by tail-folding and/or
- // non-predicated reductions to select between the updated and previous
- // reduction values.
- // Both must be combined (logical-and) and absorbed into the partial
- // reduction's Cond, and both the blend and the latch select must be
- // replaced by the partial reduction.
- VPValue *Cond = Chain.BlendMask;
- if (Cond && Chain.NegateBlendMask) {
- VPBuilder Builder(WidenRecipe);
- Cond = Builder.createNot(Cond, WidenRecipe->getDebugLoc());
- }
-
- // If the chain saw a blend, the latch select uses the
- // blend rather than the WidenRecipe directly.
- VPValue *SearchVal = WidenRecipe;
- VPBlendRecipe *BlendUser = nullptr;
- if (Chain.BlendMask) {
- BlendUser = findUserOf<VPBlendRecipe>(WidenRecipe);
- if (BlendUser)
- SearchVal = BlendUser;
- }
-
- VPValue *LatchCond = nullptr;
- VPRecipeBase *ExitRecipe = findUserOf(
- SearchVal, m_SelectLike(m_VPValue(LatchCond), m_Specific(SearchVal),
- m_Specific(RdxPhi)));
- if (!ExitRecipe) {
- ExitRecipe = findUserOf(SearchVal, m_SelectLike(m_VPValue(LatchCond),
- m_Specific(RdxPhi),
- m_Specific(SearchVal)));
- if (ExitRecipe) {
- VPBuilder Builder(WidenRecipe);
- LatchCond = Builder.createNot(LatchCond, ExitRecipe->getDebugLoc());
- }
- }
+ // through the select recipes introduced by tail-folding or predicated
+ // reductions for the BinOp's exit value.
+ VPValue *TailFoldCond = nullptr;
+ auto *ExitValue = cast_or_null<VPInstruction>(findUserOf(
+ WidenRecipe, m_Select(m_VPValue(TailFoldCond), m_Specific(WidenRecipe),
+ m_Specific(RdxPhi))));
+
+ VPValue *BlendCond = Chain.Blend ? Chain.Blend->getMask(0) : nullptr;
+ VPValue *Cond = TailFoldCond;
- // Combine the blend mask and the latch select mask.
- if (LatchCond && Cond) {
+ if (TailFoldCond && BlendCond) {
VPBuilder Builder(WidenRecipe);
- Cond =
- Builder.createLogicalAnd(LatchCond, Cond, WidenRecipe->getDebugLoc());
- } else if (LatchCond) {
- Cond = LatchCond;
+ Cond = Builder.createLogicalAnd(TailFoldCond, BlendCond,
+ WidenRecipe->getDebugLoc());
+ } else if (BlendCond) {
+ Cond = BlendCond;
}
- VPValue *ExitValue = cast_or_null<VPSingleDefRecipe>(ExitRecipe);
-
- bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
- RdxPhi->getBackedgeValue() == ExitValue ||
- (BlendUser && RdxPhi->getBackedgeValue() == BlendUser);
+ bool IsLastInChain =
+ RdxPhi->getBackedgeValue() == WidenRecipe ||
+ RdxPhi->getBackedgeValue() == ExitValue ||
+ (Chain.Blend && RdxPhi->getBackedgeValue() == Chain.Blend);
assert((!ExitValue || IsLastInChain) &&
"if we found ExitValue, it must match RdxPhi's backedge value");
@@ -6621,12 +6582,12 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
RdxUnordered{/*VFScaleFactor=*/Chain.ScaleFactor});
PartialRed->insertBefore(WidenRecipe);
- // Replace uses of the latch select (if found) with PartialRed; otherwise
+ // Replace uses of the select (if found) with PartialRed; otherwise
// the blend directly produces the reduction backedge value.
if (ExitValue)
ExitValue->replaceAllUsesWith(PartialRed);
- if (BlendUser)
- BlendUser->replaceAllUsesWith(PartialRed);
+ if (Chain.Blend)
+ Chain.Blend->replaceAllUsesWith(PartialRed);
WidenRecipe->replaceAllUsesWith(PartialRed);
// For cost-model purposes, fold this into a VPExpression.
@@ -6837,12 +6798,6 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
if (!RdxResult)
return std::nullopt;
VPValue *ExitValue = RdxResult->getOperand(0);
- VPValue *SelectedValue = nullptr;
- if (match(ExitValue, m_SelectLike(m_VPValue(), m_VPValue(SelectedValue),
- m_Specific(RedPhiR))) ||
- match(ExitValue, m_SelectLike(m_VPValue(), m_Specific(RedPhiR),
- m_VPValue(SelectedValue))))
- ExitValue = SelectedValue;
SmallVector<VPPartialReductionChain> Chain;
RecurKind RK = RedPhiR->getRecurrenceKind();
@@ -6852,42 +6807,13 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
// Work backwards from the ExitValue examining each reduction operation.
VPValue *CurrentValue = ExitValue;
while (CurrentValue != RedPhiR) {
- VPValue *LinkMask = nullptr;
- bool LinkNegateMask = false;
- if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentValue)) {
- // Look through an incoming blend. We handle both:
- // * Normalized blends ([I0, I1, M1]): introduced by inner `if`.
- // * Non-normalized blends ([I0, M0, I1, M1]): introduced by an inner
- // `if` combined with tail-folding (M0 and M1 already include the
- // header / lane mask).
- if (Blend->getNumIncomingValues() != 2)
+ VPBlendRecipe *Blend = nullptr;
+ if (auto *B = dyn_cast<VPBlendRecipe>(CurrentValue)) {
+ if (B->getNumIncomingValues() != 2)
return std::nullopt;
- VPValue *V0 = Blend->getIncomingValue(0);
- VPValue *V1 = Blend->getIncomingValue(1);
- auto IsBinOpWiden = [](VPValue *V) {
- auto *W = dyn_cast_or_null<VPWidenRecipe>(V);
- return W && Instruction::isBinaryOp(W->getOpcode());
- };
-
- // Identify which incoming is the BinOp.
- if (IsBinOpWiden(V1)) {
- // Both forms: V1 is selected when M1 is true.
- CurrentValue = V1;
- LinkMask = Blend->getMask(1);
- } else if (IsBinOpWiden(V0)) {
- CurrentValue = V0;
- if (Blend->isNormalized()) {
- // Normalized [I0, I1, M1]: V0 is default, fires when M1 is false.
- LinkMask = Blend->getMask(1);
- LinkNegateMask = true;
- } else {
- // Non-normalized [I0, M0, I1, M1]: V0 fires when M0 is true.
- LinkMask = Blend->getMask(0);
- }
- } else {
- return std::nullopt;
- }
+ Blend = B;
+ CurrentValue = B->getIncomingValue(0);
}
auto *UpdateR = dyn_cast<VPWidenRecipe>(CurrentValue);
@@ -6897,6 +6823,9 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
VPValue *Op = UpdateR->getOperand(1);
VPValue *PrevValue = UpdateR->getOperand(0);
+ if (Blend && Blend->getIncomingValue(1) != PrevValue)
+ return std::nullopt;
+
// Find the extended operand. The other operand (PrevValue) is the next link
// in the reduction chain.
std::optional<ExtendedReductionOperand> ExtendedOp =
@@ -6920,7 +6849,7 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
{UpdateR, *ExtendedOp, RK,
PrevValue == UpdateR->getOperand(0) ? 0U : 1U,
static_cast<unsigned>(PHISize.getKnownScalarFactor(ExtSrcSize)),
- LinkMask, LinkNegateMask});
+ Blend});
Chain.push_back(Link);
CurrentValue = PrevValue;
}
@@ -7012,34 +6941,8 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
return PhiR == RedPhiR;
- // Predicated reductions introduced by an 'if' inside the loop
- // will be represented by a VPBlendRecipe with two incoming values,
- // the reduction PHI and the reduction Bin Op.
- if (auto *Blend = dyn_cast<VPBlendRecipe>(U)) {
- if (Blend->getNumIncomingValues() != 2)
- return false;
-
- VPValue *V0 = Blend->getIncomingValue(0);
- VPValue *V1 = Blend->getIncomingValue(1);
- if (!((V0 == Chain.ReductionBinOp && V1 == RedPhiR) ||
- (V1 == Chain.ReductionBinOp && V0 == RedPhiR)))
- return false;
-
- // The blend's users must be the reduction PHI's backedge, the
- // compute-reduction-result, the latch select introduced
- // by tail-folding, or another scaled-reduction with the same
- // scale factor for chained reductions.
- return all_of(Blend->users(), [&](VPUser *BU) {
- if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(BU))
- return PhiR == RedPhiR;
- auto *R = dyn_cast<VPSingleDefRecipe>(BU);
- return R &&
- (Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
- match(R, m_ComputeReductionResult(m_Specific(Blend))) ||
- match(R, m_Select(m_VPValue(), m_Specific(Blend),
- m_Specific(RedPhiR))));
- });
- }
+ if (Chain.Blend == U)
+ return true;
auto *R = cast<VPSingleDefRecipe>(U);
return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
>From b7a764a497d7c78aa2f6d8b7f09849f7fc35f043 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 23 Jun 2026 10:28:55 +0000
Subject: [PATCH 07/15] Added test with reduction before predicate
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 12 +-
.../AArch64/partial-reduce-with-predicate.ll | 261 +++++++++++++++++-
2 files changed, 258 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7e992483d1933..a6ceffcb2b07e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6823,9 +6823,6 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
VPValue *Op = UpdateR->getOperand(1);
VPValue *PrevValue = UpdateR->getOperand(0);
- if (Blend && Blend->getIncomingValue(1) != PrevValue)
- return std::nullopt;
-
// Find the extended operand. The other operand (PrevValue) is the next link
// in the reduction chain.
std::optional<ExtendedReductionOperand> ExtendedOp =
@@ -6837,6 +6834,9 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
std::swap(Op, PrevValue);
}
+ if (Blend && Blend->getIncomingValue(1) != PrevValue)
+ return std::nullopt;
+
Type *ExtSrcType = ExtendedOp->ExtendA.SrcType;
TypeSize ExtSrcSize = ExtSrcType->getPrimitiveSizeInBits();
if (!PHISize.hasKnownScalarFactor(ExtSrcSize))
@@ -6941,8 +6941,10 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
return PhiR == RedPhiR;
- if (Chain.Blend == U)
- return true;
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(U))
+ return any_of(Chains, [Blend](const VPPartialReductionChain &C) {
+ return C.Blend == Blend;
+ });
auto *R = cast<VPSingleDefRecipe>(U);
return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index e22f0e4899f45..5eca50cfe958b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -907,7 +907,7 @@ exit:
define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-LABEL: define i32 @chained_pred_reduction(
-; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
@@ -935,7 +935,7 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -965,13 +965,13 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-INTERLEAVE1-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-INTERLEAVE1: [[EXIT]]:
; CHECK-INTERLEAVE1-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_2_LCSSA]]
;
; CHECK-INTERLEAVED-LABEL: define i32 @chained_pred_reduction(
-; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
@@ -1013,7 +1013,7 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE8]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE4]], <vscale x 16 x i32> [[TMP17]])
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
@@ -1044,13 +1044,13 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-INTERLEAVED-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-INTERLEAVED: [[EXIT]]:
; CHECK-INTERLEAVED-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_2_LCSSA]]
;
; CHECK-MAXBW-LABEL: define i32 @chained_pred_reduction(
-; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
@@ -1078,7 +1078,7 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -1108,7 +1108,7 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-MAXBW-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-MAXBW: [[EXIT]]:
; CHECK-MAXBW-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-MAXBW-NEXT: ret i32 [[SUM_2_LCSSA]]
@@ -1133,7 +1133,6 @@ if.then:
for.inc:
%sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
- ; Chain to result of blend:
%b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv
%bval = load i8, ptr %b.gep, align 1
%bconv = zext i8 %bval to i32
@@ -1146,4 +1145,246 @@ exit:
ret i32 %sum.2
}
+define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @reduction_before_pred(
+; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1:%.*]] = add nsw i32 [[SUM]], [[BCONV]]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_2]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_1]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @reduction_before_pred(
+; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP6]])
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP9]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD4]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD5]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP15]])
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD6]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = select <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i32> [[TMP16]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE8]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE3]], <vscale x 16 x i32> [[TMP17]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
+; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-INTERLEAVED-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[SUM_1:%.*]] = add nsw i32 [[SUM]], [[BCONV]]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_2]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_1]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @reduction_before_pred(
+; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP4]])
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-MAXBW: [[FOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-MAXBW-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-MAXBW-NEXT: [[SUM_1:%.*]] = add nsw i32 [[SUM]], [[BCONV]]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_2]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_1]], %[[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-MAXBW: [[EXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ]
+
+ %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv
+ %bval = load i8, ptr %b.gep, align 1
+ %bconv = zext i8 %bval to i32
+ %sum.1 = add nsw i32 %sum, %bconv
+
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum.1, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.2 = phi i32 [ %add, %if.then ], [ %sum.1, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.2
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
>From 9abc24ae993d2e52dfcf2c6f114a1a22edde1c6a Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 23 Jun 2026 11:03:27 +0000
Subject: [PATCH 08/15] fix structured binding error
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index a6ceffcb2b07e..be59bf9e06d67 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6937,7 +6937,7 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
Chains.clear();
break;
}
- auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
+ auto UseIsValid = [&, RedPhiR = RedPhiR, &Chains = Chains](VPUser *U) {
if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
return PhiR == RedPhiR;
>From 2bbf35f526a6cdd55c696eb2137e1f272166c8aa Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 23 Jun 2026 14:17:45 +0000
Subject: [PATCH 09/15] Fixups to VPlanTransforms
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 27 ++++++++++++-------
1 file changed, 18 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index be59bf9e06d67..acd24661295c2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6567,7 +6567,7 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
bool IsLastInChain =
RdxPhi->getBackedgeValue() == WidenRecipe ||
RdxPhi->getBackedgeValue() == ExitValue ||
- (Chain.Blend && RdxPhi->getBackedgeValue() == Chain.Blend);
+ RdxPhi->getBackedgeValue() == Chain.Blend;
assert((!ExitValue || IsLastInChain) &&
"if we found ExitValue, it must match RdxPhi's backedge value");
@@ -6798,6 +6798,7 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
if (!RdxResult)
return std::nullopt;
VPValue *ExitValue = RdxResult->getOperand(0);
+ match(ExitValue, m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
SmallVector<VPPartialReductionChain> Chain;
RecurKind RK = RedPhiR->getRecurrenceKind();
@@ -6811,9 +6812,13 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
if (auto *B = dyn_cast<VPBlendRecipe>(CurrentValue)) {
if (B->getNumIncomingValues() != 2)
return std::nullopt;
+ assert(!B->isNormalized() && "Expect Blend not to be normalized.");
Blend = B;
CurrentValue = B->getIncomingValue(0);
+
+ if (!CurrentValue->hasOneUse())
+ return std::nullopt;
}
auto *UpdateR = dyn_cast<VPWidenRecipe>(CurrentValue);
@@ -6834,6 +6839,8 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
std::swap(Op, PrevValue);
}
+ // Look for VPBlend(cond_reduce(PrevValue, Op), PrevValue), where
+ // cond_reduce is equal to CurrentValue.
if (Blend && Blend->getIncomingValue(1) != PrevValue)
return std::nullopt;
@@ -6882,13 +6889,16 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
if (ChainsByPhi.empty())
return;
- // Build set of partial reduction operations for extend user validation and
- // a map of reduction bin ops to their scale factors for scale validation.
+ // Build set of partial reduction operations and blends for extend user validation
+ // and a map of reduction bin ops to their scale factors for scale validation.
SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
+ SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
for (const auto &[_, Chains] : ChainsByPhi)
for (const VPPartialReductionChain &Chain : Chains) {
PartialReductionOps.insert(Chain.ExtendedOp.ExtendsUser);
+ if (Chain.Blend)
+ PartialReductionBlends.insert(Chain.Blend);
ScaledReductionMap[Chain.ReductionBinOp] = Chain.ScaleFactor;
}
@@ -6937,16 +6947,15 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
Chains.clear();
break;
}
- auto UseIsValid = [&, RedPhiR = RedPhiR, &Chains = Chains](VPUser *U) {
+ auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
return PhiR == RedPhiR;
- if (auto *Blend = dyn_cast<VPBlendRecipe>(U))
- return any_of(Chains, [Blend](const VPPartialReductionChain &C) {
- return C.Blend == Blend;
- });
-
auto *R = cast<VPSingleDefRecipe>(U);
+
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(R))
+ return Blend == Chain.Blend || PartialReductionBlends.contains(Blend);
+
return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
match(R, m_ComputeReductionResult(
m_Specific(Chain.ReductionBinOp))) ||
>From 99771259f57454f6cdc317a5a8aede63879678f7 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 23 Jun 2026 15:57:35 +0000
Subject: [PATCH 10/15] Add negative tests
---
.../AArch64/partial-reduce-with-predicate.ll | 1267 +++++++++--------
1 file changed, 663 insertions(+), 604 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index 5eca50cfe958b..095e81dacdef6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVE1
; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVED
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-MAXBW
+; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -tail-folding-policy=must-fold-tail -S < %s | FileCheck %s --check-prefixes=CHECK-TAILFOLD
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"
@@ -9,182 +9,160 @@ target triple = "aarch64-none-unknown-elf"
define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-LABEL: define i32 @pred_reduction(
; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-INTERLEAVE1-NEXT: [[FOR_BODY_PREHEADER:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
-; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
-; CHECK-INTERLEAVE1: [[FOR_COND_CLEANUP]]:
-; CHECK-INTERLEAVE1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVE1-NEXT: [[SUM_08:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDVARS_IV]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP11]], 0
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
; CHECK-INTERLEAVE1: [[IF_THEN]]:
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[INDVARS_IV]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[TMP12]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_08]], [[CONV]]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
; CHECK-INTERLEAVE1: [[FOR_INC]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_08]], %[[FOR_COND_CLEANUP]] ]
-; CHECK-INTERLEAVE1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK-INTERLEAVE1: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
;
; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction(
; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-INTERLEAVED-NEXT: [[FOR_BODY_PREHEADER:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP11]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP12]])
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP13]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP14]])
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP12]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP13]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
-; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-INTERLEAVED: [[SCALAR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
-; CHECK-INTERLEAVED: [[FOR_COND_CLEANUP]]:
-; CHECK-INTERLEAVED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVED-NEXT: [[SUM_08:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDVARS_IV]]
-; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP17]], 0
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
; CHECK-INTERLEAVED: [[IF_THEN]]:
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[INDVARS_IV]]
-; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[TMP18]] to i32
-; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_08]], [[CONV]]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
; CHECK-INTERLEAVED: [[FOR_INC]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_08]], %[[FOR_COND_CLEANUP]] ]
-; CHECK-INTERLEAVED-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK-INTERLEAVED: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
;
-; CHECK-MAXBW-LABEL: define i32 @pred_reduction(
-; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-MAXBW-NEXT: [[FOR_BODY_PREHEADER:.*]]:
-; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-MAXBW: [[VECTOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
-; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
-; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
-; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-MAXBW: [[SCALAR_PH]]:
-; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
-; CHECK-MAXBW-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
-; CHECK-MAXBW: [[FOR_COND_CLEANUP]]:
-; CHECK-MAXBW-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-MAXBW-NEXT: [[SUM_08:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDVARS_IV]]
-; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP11]], 0
-; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-MAXBW: [[IF_THEN]]:
-; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[INDVARS_IV]]
-; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-MAXBW-NEXT: [[CONV:%.*]] = zext i8 [[TMP12]] to i32
-; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_08]], [[CONV]]
-; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
-; CHECK-MAXBW: [[FOR_INC]]:
-; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_08]], %[[FOR_COND_CLEANUP]] ]
-; CHECK-MAXBW-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK-MAXBW: [[FOR_COND_CLEANUP_LOOPEXIT]]:
-; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+; CHECK-TAILFOLD-LABEL: define i32 @pred_reduction(
+; CHECK-TAILFOLD-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-TAILFOLD-NEXT: [[ENTRY:.*:]]
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TAILFOLD-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP4:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP3]], <vscale x 16 x i1> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-TAILFOLD-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-TAILFOLD-NEXT: [[TMP8]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TAILFOLD-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP8]])
+; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; CHECK-TAILFOLD: [[EXIT]]:
+; CHECK-TAILFOLD-NEXT: ret i32 [[TMP11]]
;
entry:
br label %for.body
@@ -223,31 +201,31 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
-; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-INTERLEAVE1: [[FOR_BODY]]:
; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
@@ -268,7 +246,7 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-INTERLEAVE1: [[EXIT]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
;
; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction_sext(
@@ -279,42 +257,42 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP11]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP12]])
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP13]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP14]])
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP12]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP13]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
-; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-INTERLEAVED: [[SCALAR_PH]]:
; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-INTERLEAVED: [[FOR_BODY]]:
; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
@@ -335,64 +313,42 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-INTERLEAVED: [[EXIT]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
;
-; CHECK-MAXBW-LABEL: define i32 @pred_reduction_sext(
-; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
-; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-MAXBW: [[VECTOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
-; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
-; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
-; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-MAXBW: [[SCALAR_PH]]:
-; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-MAXBW: [[FOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-MAXBW: [[IF_THEN]]:
-; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-MAXBW-NEXT: [[CONV:%.*]] = sext i8 [[VAL]] to i32
-; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
-; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
-; CHECK-MAXBW: [[FOR_INC]]:
-; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK-MAXBW: [[EXIT]]:
-; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
-; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+; CHECK-TAILFOLD-LABEL: define i32 @pred_reduction_sext(
+; CHECK-TAILFOLD-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-NEXT: [[ENTRY:.*:]]
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TAILFOLD-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP4:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP3]], <vscale x 16 x i1> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-TAILFOLD-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-TAILFOLD-NEXT: [[TMP8]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
+; CHECK-TAILFOLD-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP8]])
+; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; CHECK-TAILFOLD: [[EXIT]]:
+; CHECK-TAILFOLD-NEXT: ret i32 [[TMP11]]
;
entry:
br label %for.body
@@ -431,35 +387,35 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
-; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP9]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP10]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-INTERLEAVE1: [[FOR_BODY]]:
; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
@@ -470,12 +426,12 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
; CHECK-INTERLEAVE1: [[IF_THEN]]:
; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[VA:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[VB:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[CONV_A:%.*]] = zext i8 [[VA]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[CONV_B:%.*]] = zext i8 [[VB]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV_A]], [[CONV_B]]
+; CHECK-INTERLEAVE1-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
; CHECK-INTERLEAVE1: [[FOR_INC]]:
@@ -484,7 +440,7 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-INTERLEAVE1: [[EXIT]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
;
; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction_dotprod(
@@ -495,50 +451,50 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP11]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP13]], [[TMP14]]
-; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP15]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP16]])
-; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP17]], [[TMP18]]
-; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP19]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP20]])
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP12]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP15]])
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP16]], [[TMP17]]
+; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP18]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP19]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
-; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-INTERLEAVED: [[SCALAR_PH]]:
; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-INTERLEAVED: [[FOR_BODY]]:
; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
@@ -549,12 +505,12 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
; CHECK-INTERLEAVED: [[IF_THEN]]:
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[VA:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[VB:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
-; CHECK-INTERLEAVED-NEXT: [[CONV_A:%.*]] = zext i8 [[VA]] to i32
-; CHECK-INTERLEAVED-NEXT: [[CONV_B:%.*]] = zext i8 [[VB]] to i32
-; CHECK-INTERLEAVED-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV_A]], [[CONV_B]]
+; CHECK-INTERLEAVED-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVED-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
+; CHECK-INTERLEAVED-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
; CHECK-INTERLEAVED: [[FOR_INC]]:
@@ -563,72 +519,46 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-INTERLEAVED: [[EXIT]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP22]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP21]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
;
-; CHECK-MAXBW-LABEL: define i32 @pred_reduction_dotprod(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
-; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-MAXBW: [[VECTOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
-; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
-; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
-; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
-; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
-; CHECK-MAXBW-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-MAXBW: [[SCALAR_PH]]:
-; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-MAXBW: [[FOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-MAXBW: [[IF_THEN]]:
-; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[VA:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-MAXBW-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[VB:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
-; CHECK-MAXBW-NEXT: [[CONV_A:%.*]] = zext i8 [[VA]] to i32
-; CHECK-MAXBW-NEXT: [[CONV_B:%.*]] = zext i8 [[VB]] to i32
-; CHECK-MAXBW-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV_A]], [[CONV_B]]
-; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
-; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
-; CHECK-MAXBW: [[FOR_INC]]:
-; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK-MAXBW: [[EXIT]]:
-; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
-; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+; CHECK-TAILFOLD-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-TAILFOLD-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-NEXT: [[ENTRY:.*:]]
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TAILFOLD-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP4:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP3]], <vscale x 16 x i1> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-TAILFOLD-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD2]] to <vscale x 16 x i32>
+; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-TAILFOLD-NEXT: [[TMP10:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP9]], <vscale x 16 x i32> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP10]])
+; CHECK-TAILFOLD-NEXT: [[TMP11]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TAILFOLD-NEXT: [[TMP12:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
+; CHECK-TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP11]])
+; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; CHECK-TAILFOLD: [[EXIT]]:
+; CHECK-TAILFOLD-NEXT: ret i32 [[TMP14]]
;
entry:
br label %for.body
@@ -671,36 +601,36 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
-; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP9]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP10]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-INTERLEAVE1-NEXT: [[TMP14:%.*]] = sub i32 0, [[TMP13]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = sub i32 0, [[TMP12]]
; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-INTERLEAVE1: [[FOR_BODY]]:
; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
@@ -725,7 +655,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK-INTERLEAVE1: [[EXIT]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
;
; CHECK-INTERLEAVED-LABEL: define i32 @pred_sub_reduction(
@@ -736,51 +666,51 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP11]], i64 [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP13]], [[TMP14]]
-; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP15]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP16]])
-; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP17]], [[TMP18]]
-; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP19]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP20]])
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
-; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP12]], [[TMP13]]
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP15]])
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP16]], [[TMP17]]
+; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP18]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP19]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
-; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = sub i32 0, [[TMP22]]
+; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = sub i32 0, [[TMP21]]
; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
; CHECK-INTERLEAVED: [[SCALAR_PH]]:
; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP23]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
; CHECK-INTERLEAVED: [[FOR_BODY]]:
; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
@@ -805,73 +735,46 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
; CHECK-INTERLEAVED: [[EXIT]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP23]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP22]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
;
-; CHECK-MAXBW-LABEL: define i32 @pred_sub_reduction(
-; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
-; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-MAXBW: [[VECTOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
-; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
-; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
-; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
-; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
-; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
-; CHECK-MAXBW-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-MAXBW-NEXT: [[TMP14:%.*]] = sub i32 0, [[TMP13]]
-; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-MAXBW: [[SCALAR_PH]]:
-; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-MAXBW: [[FOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-MAXBW: [[IF_THEN]]:
-; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-MAXBW-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
-; CHECK-MAXBW-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
-; CHECK-MAXBW-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
-; CHECK-MAXBW-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
-; CHECK-MAXBW-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM]], [[MUL]]
-; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
-; CHECK-MAXBW: [[FOR_INC]]:
-; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[SUB]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK-MAXBW: [[EXIT]]:
-; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
-; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+; CHECK-TAILFOLD-LABEL: define i32 @pred_sub_reduction(
+; CHECK-TAILFOLD-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-NEXT: [[ENTRY:.*:]]
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TAILFOLD-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP4:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP3]], <vscale x 16 x i1> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-TAILFOLD-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD2]] to <vscale x 16 x i32>
+; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP7]], [[TMP8]]
+; CHECK-TAILFOLD-NEXT: [[TMP10:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP9]], <vscale x 16 x i32> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP10]])
+; CHECK-TAILFOLD-NEXT: [[TMP11]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TAILFOLD-NEXT: [[TMP12:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
+; CHECK-TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP11]])
+; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; CHECK-TAILFOLD: [[EXIT]]:
+; CHECK-TAILFOLD-NEXT: ret i32 [[TMP14]]
;
entry:
br label %for.body
@@ -1049,69 +952,43 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-INTERLEAVED-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_2_LCSSA]]
;
-; CHECK-MAXBW-LABEL: define i32 @chained_pred_reduction(
-; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
-; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
-; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-MAXBW: [[VECTOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
-; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
-; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
-; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
-; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
-; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
-; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
-; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
-; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
-; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-MAXBW: [[SCALAR_PH]]:
-; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-MAXBW: [[FOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
-; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-MAXBW: [[IF_THEN]]:
-; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-MAXBW-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
-; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
-; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
-; CHECK-MAXBW: [[FOR_INC]]:
-; CHECK-MAXBW-NEXT: [[SUM_1:%.*]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
-; CHECK-MAXBW-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
-; CHECK-MAXBW-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
-; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK-MAXBW: [[EXIT]]:
-; CHECK-MAXBW-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
-; CHECK-MAXBW-NEXT: ret i32 [[SUM_2_LCSSA]]
+; CHECK-TAILFOLD-LABEL: define i32 @chained_pred_reduction(
+; CHECK-TAILFOLD-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-NEXT: [[ENTRY:.*:]]
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TAILFOLD-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP4:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP3]], <vscale x 16 x i1> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-TAILFOLD-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-TAILFOLD-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD2]] to <vscale x 16 x i32>
+; CHECK-TAILFOLD-NEXT: [[TMP10:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[TMP9]], <vscale x 16 x i32> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE3]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP10]])
+; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TAILFOLD-NEXT: [[TMP11:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
+; CHECK-TAILFOLD-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE3]])
+; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; CHECK-TAILFOLD: [[EXIT]]:
+; CHECK-TAILFOLD-NEXT: ret i32 [[TMP13]]
;
entry:
br label %for.body
@@ -1202,7 +1079,7 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
; CHECK-INTERLEAVE1: [[FOR_INC]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_2]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_1]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM_2]] = phi i32 [ [[SUM_1]], %[[FOR_BODY]] ], [ [[ADD]], %[[IF_THEN]] ]
; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -1281,7 +1158,7 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
; CHECK-INTERLEAVED: [[FOR_INC]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_2]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_1]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM_2]] = phi i32 [ [[SUM_1]], %[[FOR_BODY]] ], [ [[ADD]], %[[IF_THEN]] ]
; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
@@ -1289,69 +1166,43 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-INTERLEAVED-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_2_LCSSA]]
;
-; CHECK-MAXBW-LABEL: define i32 @reduction_before_pred(
-; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
-; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
-; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-MAXBW: [[VECTOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
-; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
-; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP4]])
-; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD1]], zeroinitializer
-; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
-; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
-; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
-; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
-; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
-; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-MAXBW: [[SCALAR_PH]]:
-; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-MAXBW: [[FOR_BODY]]:
-; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
-; CHECK-MAXBW-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
-; CHECK-MAXBW-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
-; CHECK-MAXBW-NEXT: [[SUM_1:%.*]] = add nsw i32 [[SUM]], [[BCONV]]
-; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-MAXBW: [[IF_THEN]]:
-; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-MAXBW-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-MAXBW-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
-; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
-; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
-; CHECK-MAXBW: [[FOR_INC]]:
-; CHECK-MAXBW-NEXT: [[SUM_2]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_1]], %[[FOR_BODY]] ]
-; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK-MAXBW: [[EXIT]]:
-; CHECK-MAXBW-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
-; CHECK-MAXBW-NEXT: ret i32 [[SUM_2_LCSSA]]
+; CHECK-TAILFOLD-LABEL: define i32 @reduction_before_pred(
+; CHECK-TAILFOLD-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-TAILFOLD-NEXT: [[ENTRY:.*:]]
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_PH:.*]]
+; CHECK-TAILFOLD: [[VECTOR_PH]]:
+; CHECK-TAILFOLD-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-TAILFOLD-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[N]])
+; CHECK-TAILFOLD-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-TAILFOLD: [[VECTOR_BODY]]:
+; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP3]])
+; CHECK-TAILFOLD-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP4]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]], zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP6:%.*]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i1> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-TAILFOLD-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD2]] to <vscale x 16 x i32>
+; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE3:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-TAILFOLD-NEXT: [[TMP10]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[PARTIAL_REDUCE3]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
+; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
+; CHECK-TAILFOLD-NEXT: [[TMP11:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
+; CHECK-TAILFOLD-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
+; CHECK-TAILFOLD-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
+; CHECK-TAILFOLD-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP10]])
+; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
+; CHECK-TAILFOLD: [[EXIT]]:
+; CHECK-TAILFOLD-NEXT: ret i32 [[TMP13]]
;
entry:
br label %for.body
@@ -1377,6 +1228,165 @@ if.then:
%add = add nsw i32 %sum.1, %conv
br label %for.inc
+for.inc:
+ %sum.2 = phi i32 [ %sum.1, %for.body ], [ %add, %if.then ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.2
+}
+
+; Negative test - expect not to generate a partial reduction as phi
+; has two update values
+define i32 @partial_reduce_if_else(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @partial_reduce_if_else(
+; CHECK-INTERLEAVE1-NOT: llvm.vector.partial.reduce
+; CHECK-INTERLEAVE1: call i32 @llvm.vector.reduce.add
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @partial_reduce_if_else(
+; CHECK-INTERLEAVED-NOT: llvm.vector.partial.reduce
+; CHECK-INTERLEAVED: call i32 @llvm.vector.reduce.add
+;
+; CHECK-TAILFOLD-LABEL: define i32 @partial_reduce_if_else(
+; CHECK-TAILFOLD-NOT: llvm.vector.partial.reduce
+; CHECK-TAILFOLD: call i32 @llvm.vector.reduce.add
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.next, %for.inc ]
+
+ %arrayidx.cond = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx.cond, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %if.else, label %if.then
+
+if.then:
+ %arrayidx.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %aval = load i8, ptr %arrayidx.a, align 1
+ %a.ext = zext i8 %aval to i32
+ %add.a = add nsw i32 %sum, %a.ext
+ br label %for.inc
+
+if.else:
+ %arrayidx.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %bval = load i8, ptr %arrayidx.b, align 1
+ %b.ext = zext i8 %bval to i32
+ %add.b = add nsw i32 %sum, %b.ext
+ br label %for.inc
+
+for.inc:
+ %sum.next = phi i32 [ %add.a, %if.then ], [ %add.b, %if.else ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.next
+}
+
+; Negative test - expect not to generate the partial reduction as
+; there are more than two incomng values to the predicated phi.
+define i32 @partial_reduce_three_incoming(ptr %a, ptr %b, ptr %cond_a, ptr %cond_b, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @partial_reduce_three_incoming(
+; CHECK-INTERLEAVE1-NOT: llvm.vector.partial.reduce
+; CHECK-INTERLEAVE1: call i32 @llvm.vector.reduce.add
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @partial_reduce_three_incoming(
+; CHECK-INTERLEAVED-NOT: llvm.vector.partial.reduce
+; CHECK-INTERLEAVED: call i32 @llvm.vector.reduce.add
+;
+; CHECK-TAILFOLD-LABEL: define i32 @partial_reduce_three_incoming(
+; CHECK-TAILFOLD-NOT: llvm.vector.partial.reduce
+; CHECK-TAILFOLD: call i32 @llvm.vector.reduce.add
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.next, %for.inc ]
+
+ %arrayidx.cond.a = getelementptr inbounds nuw i8, ptr %cond_a, i64 %iv
+ %ca = load i8, ptr %arrayidx.cond.a, align 1
+ %cond.a = icmp ne i8 %ca, 0
+ br i1 %cond.a, label %if.then, label %if.else
+
+if.then:
+ %arrayidx.a = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %aval = load i8, ptr %arrayidx.a, align 1
+ %a.ext = zext i8 %aval to i32
+ %add.a = add nsw i32 %sum, %a.ext
+ br label %for.inc
+
+if.else:
+ %arrayidx.cond.b = getelementptr inbounds nuw i8, ptr %cond_b, i64 %iv
+ %cb = load i8, ptr %arrayidx.cond.b, align 1
+ %cond.b = icmp ne i8 %cb, 0
+ br i1 %cond.b, label %if.else.then, label %if.else.end
+
+if.else.then:
+ %arrayidx.b = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %bval = load i8, ptr %arrayidx.b, align 1
+ %b.ext = zext i8 %bval to i32
+ %add.b = add nsw i32 %sum, %b.ext
+ br label %for.inc
+
+if.else.end:
+ br label %for.inc
+
+for.inc:
+ %sum.next = phi i32 [ %add.a, %if.then ],
+ [ %add.b, %if.else.then ],
+ [ %sum, %if.else.end ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.next
+}
+
+; Negative test - chained reduction with mixed scale factors
+define i32 @partial_reduce_mixed_scale_factor(ptr %a8, ptr %b16,
+ ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @partial_reduce_mixed_scale_factor(
+; CHECK-INTERLEAVE1-NOT: llvm.vector.partial.reduce
+; CHECK-INTERLEAVE1: call i32 @llvm.vector.reduce.add
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @partial_reduce_mixed_scale_factor(
+; CHECK-INTERLEAVED-NOT: llvm.vector.partial.reduce
+; CHECK-INTERLEAVED: call i32 @llvm.vector.reduce.add
+;
+; CHECK-TAILFOLD-LABEL: define i32 @partial_reduce_mixed_scale_factor(
+; CHECK-TAILFOLD-NOT: llvm.vector.partial.reduce
+; CHECK-TAILFOLD: call i32 @llvm.vector.reduce.add
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ]
+
+ %a.gep = getelementptr inbounds nuw i8, ptr %a8, i64 %iv
+ %aval = load i8, ptr %a.gep, align 1
+ %a.ext = zext i8 %aval to i32
+ %sum.1 = add nsw i32 %sum, %a.ext
+
+ %cond.gep = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %cond.gep, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %b.gep = getelementptr inbounds nuw i16, ptr %b16, i64 %iv
+ %bval = load i16, ptr %b.gep, align 2
+ %b.ext = zext i16 %bval to i32
+ %add = add nsw i32 %sum.1, %b.ext
+ br label %for.inc
+
for.inc:
%sum.2 = phi i32 [ %add, %if.then ], [ %sum.1, %for.body ]
%iv.next = add nuw nsw i64 %iv, 1
@@ -1387,4 +1397,53 @@ exit:
ret i32 %sum.2
}
+; Negative test - the incoming value doesn't match the accumulator
+; the passthrough value is %sum, not the previous chain value %sum.1.
+define i32 @partial_reduce_bad_passthrough(ptr %src, ptr noalias %src_b,
+ ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @partial_reduce_bad_passthrough(
+; CHECK-INTERLEAVE1-NOT: llvm.vector.partial.reduce
+; CHECK-INTERLEAVE1: call i32 @llvm.vector.reduce.add
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @partial_reduce_bad_passthrough(
+; CHECK-INTERLEAVED-NOT: llvm.vector.partial.reduce
+; CHECK-INTERLEAVED: call i32 @llvm.vector.reduce.add
+;
+; CHECK-TAILFOLD-LABEL: define i32 @partial_reduce_bad_passthrough(
+; CHECK-TAILFOLD-NOT: llvm.vector.partial.reduce
+; CHECK-TAILFOLD: call i32 @llvm.vector.reduce.add
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ]
+
+ %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv
+ %bval = load i8, ptr %b.gep, align 1
+ %bconv = zext i8 %bval to i32
+ %sum.1 = add nsw i32 %sum, %bconv
+
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum.1, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.2 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.2
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
>From a98c0e188706207744ad2b0037c9cfa6979454ec Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 24 Jun 2026 08:49:33 +0000
Subject: [PATCH 11/15] fix formatting
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index acd24661295c2..b4acbd5260966 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6564,10 +6564,9 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
Cond = BlendCond;
}
- bool IsLastInChain =
- RdxPhi->getBackedgeValue() == WidenRecipe ||
- RdxPhi->getBackedgeValue() == ExitValue ||
- RdxPhi->getBackedgeValue() == Chain.Blend;
+ bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
+ RdxPhi->getBackedgeValue() == ExitValue ||
+ RdxPhi->getBackedgeValue() == Chain.Blend;
assert((!ExitValue || IsLastInChain) &&
"if we found ExitValue, it must match RdxPhi's backedge value");
@@ -6889,8 +6888,9 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
if (ChainsByPhi.empty())
return;
- // Build set of partial reduction operations and blends for extend user validation
- // and a map of reduction bin ops to their scale factors for scale validation.
+ // Build set of partial reduction operations and blends for extend user
+ // validation and a map of reduction bin ops to their scale factors for scale
+ // validation.
SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
>From c93a962591ab2cb90123e36acd683ed5a56b4864 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 24 Jun 2026 16:26:27 +0000
Subject: [PATCH 12/15] Rm interleave test and comment fixups
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 10 +-
.../AArch64/partial-reduce-with-predicate.ll | 1104 +++++------------
2 files changed, 291 insertions(+), 823 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b4acbd5260966..302b51900e233 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6581,8 +6581,6 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
RdxUnordered{/*VFScaleFactor=*/Chain.ScaleFactor});
PartialRed->insertBefore(WidenRecipe);
- // Replace uses of the select (if found) with PartialRed; otherwise
- // the blend directly produces the reduction backedge value.
if (ExitValue)
ExitValue->replaceAllUsesWith(PartialRed);
if (Chain.Blend)
@@ -6839,7 +6837,8 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
}
// Look for VPBlend(cond_reduce(PrevValue, Op), PrevValue), where
- // cond_reduce is equal to CurrentValue.
+ // cond_reduce is equal to CurrentValue. This can be lowered as
+ // a conditional reduction by hoisting the select to the inputs.
if (Blend && Blend->getIncomingValue(1) != PrevValue)
return std::nullopt;
@@ -6888,9 +6887,8 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
if (ChainsByPhi.empty())
return;
- // Build set of partial reduction operations and blends for extend user
- // validation and a map of reduction bin ops to their scale factors for scale
- // validation.
+ // Build set of partial reduction operations and blends for user validation
+ // and a map of reduction bin ops to their scale factors for scale validation.
SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index 095e81dacdef6..28aec1c2e0ed5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -1,134 +1,53 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
-; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVE1
-; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVED
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "^scalar.ph" --version 6
+; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK
; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -tail-folding-policy=must-fold-tail -S < %s | FileCheck %s --check-prefixes=CHECK-TAILFOLD
target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"
define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @pred_reduction(
-; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
-; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[FOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVE1: [[IF_THEN]]:
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVE1: [[FOR_INC]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK-INTERLEAVE1: [[EXIT]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction(
-; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
-; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP12]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP13]])
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
-; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVED: [[SCALAR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[FOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVED: [[IF_THEN]]:
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
-; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVED: [[FOR_INC]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK-INTERLEAVED: [[EXIT]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+; CHECK-LABEL: define i32 @pred_reduction(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP12]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP13]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
+; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
;
; CHECK-TAILFOLD-LABEL: define i32 @pred_reduction(
; CHECK-TAILFOLD-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
@@ -193,128 +112,48 @@ exit:
}
define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @pred_reduction_sext(
-; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
-; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[FOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVE1: [[IF_THEN]]:
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = sext i8 [[VAL]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVE1: [[FOR_INC]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK-INTERLEAVE1: [[EXIT]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction_sext(
-; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
-; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP12]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP13]])
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVED: [[SCALAR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[FOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVED: [[IF_THEN]]:
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = sext i8 [[VAL]] to i32
-; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVED: [[FOR_INC]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK-INTERLEAVED: [[EXIT]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+; CHECK-LABEL: define i32 @pred_reduction_sext(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-NEXT: [[TMP12:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP12]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP13]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
+; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
;
; CHECK-TAILFOLD-LABEL: define i32 @pred_reduction_sext(
; CHECK-TAILFOLD-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
@@ -379,148 +218,56 @@ exit:
}
define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @pred_reduction_dotprod(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP7]], [[TMP8]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP9]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP10]])
-; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP12]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[FOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVE1: [[IF_THEN]]:
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
-; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVE1: [[FOR_INC]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK-INTERLEAVE1: [[EXIT]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP12]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction_dotprod(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
-; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP12]], [[TMP13]]
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP15]])
-; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP16]], [[TMP17]]
-; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP18]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP19]])
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
-; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
-; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVED: [[SCALAR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP21]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[FOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVED: [[IF_THEN]]:
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
-; CHECK-INTERLEAVED-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVED-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
-; CHECK-INTERLEAVED-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
-; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVED: [[FOR_INC]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK-INTERLEAVED: [[EXIT]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP21]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+; CHECK-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP14:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP12]], [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP15]])
+; CHECK-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP16]], [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP18]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP19]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
+; CHECK-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
;
; CHECK-TAILFOLD-LABEL: define i32 @pred_reduction_dotprod(
; CHECK-TAILFOLD-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
@@ -593,150 +340,57 @@ exit:
}
define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @pred_sub_reduction(
-; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP7]], [[TMP8]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP9]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP10]])
-; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
-; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = sub i32 0, [[TMP12]]
-; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[FOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVE1: [[IF_THEN]]:
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
-; CHECK-INTERLEAVE1-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM]], [[MUL]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVE1: [[FOR_INC]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[SUB]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK-INTERLEAVE1: [[EXIT]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @pred_sub_reduction(
-; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
-; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP12]], [[TMP13]]
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP15]])
-; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP16]], [[TMP17]]
-; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP18]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP19]])
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
-; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
-; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = sub i32 0, [[TMP21]]
-; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVED: [[SCALAR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[FOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVED: [[IF_THEN]]:
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
-; CHECK-INTERLEAVED-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
-; CHECK-INTERLEAVED-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
-; CHECK-INTERLEAVED-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
-; CHECK-INTERLEAVED-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM]], [[MUL]]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVED: [[FOR_INC]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[SUB]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK-INTERLEAVED: [[EXIT]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP22]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+; CHECK-LABEL: define i32 @pred_sub_reduction(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP14:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP12]], [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP15]])
+; CHECK-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP18:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP16]], [[TMP17]]
+; CHECK-NEXT: [[TMP19:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP18]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP19]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
+; CHECK-NEXT: [[TMP21:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[TMP22:%.*]] = sub i32 0, [[TMP21]]
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
;
; CHECK-TAILFOLD-LABEL: define i32 @pred_sub_reduction(
; CHECK-TAILFOLD-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
@@ -809,148 +463,56 @@ exit:
}
define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @chained_pred_reduction(
-; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
-; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
-; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[FOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVE1: [[IF_THEN]]:
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVE1: [[FOR_INC]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1:%.*]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
-; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK-INTERLEAVE1: [[EXIT]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_2_LCSSA]]
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @chained_pred_reduction(
-; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
-; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP12]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP13]])
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP14]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 16 x i8>, ptr [[TMP15]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD5]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP16]])
-; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD6]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE8]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE4]], <vscale x 16 x i32> [[TMP17]])
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
-; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVED: [[SCALAR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[FOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVED: [[IF_THEN]]:
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
-; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVED: [[FOR_INC]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_1:%.*]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
-; CHECK-INTERLEAVED-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
-; CHECK-INTERLEAVED-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
-; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK-INTERLEAVED: [[EXIT]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_2_LCSSA]]
+; CHECK-LABEL: define i32 @chained_pred_reduction(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP10:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP12]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP13]])
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP14]], align 1
+; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 16 x i8>, ptr [[TMP15]], align 1
+; CHECK-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD5]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[PARTIAL_REDUCE7]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP16]])
+; CHECK-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD6]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[PARTIAL_REDUCE8]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE4]], <vscale x 16 x i32> [[TMP17]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
+; CHECK-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
;
; CHECK-TAILFOLD-LABEL: define i32 @chained_pred_reduction(
; CHECK-TAILFOLD-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
@@ -1023,148 +585,56 @@ exit:
}
define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @reduction_before_pred(
-; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP4]])
-; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD1]], zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
-; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
-; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
-; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVE1: [[FOR_BODY]]:
-; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVE1-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[SUM_1:%.*]] = add nsw i32 [[SUM]], [[BCONV]]
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVE1: [[IF_THEN]]:
-; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
-; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
-; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVE1: [[FOR_INC]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_2]] = phi i32 [ [[SUM_1]], %[[FOR_BODY]] ], [ [[ADD]], %[[IF_THEN]] ]
-; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK-INTERLEAVE1: [[EXIT]]:
-; CHECK-INTERLEAVE1-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_2_LCSSA]]
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @reduction_before_pred(
-; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
-; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
-; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
-; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP6]])
-; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP7]])
-; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
-; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP9]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD4]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD5]], zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP2]]
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i8> poison)
-; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP15]])
-; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD6]] to <vscale x 16 x i32>
-; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = select <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i32> [[TMP16]], <vscale x 16 x i32> zeroinitializer
-; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE8]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE3]], <vscale x 16 x i32> [[TMP17]])
-; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
-; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
-; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
-; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
-; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
-; CHECK-INTERLEAVED: [[SCALAR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
-; CHECK-INTERLEAVED: [[FOR_BODY]]:
-; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
-; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
-; CHECK-INTERLEAVED-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
-; CHECK-INTERLEAVED-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
-; CHECK-INTERLEAVED-NEXT: [[SUM_1:%.*]] = add nsw i32 [[SUM]], [[BCONV]]
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
-; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
-; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
-; CHECK-INTERLEAVED: [[IF_THEN]]:
-; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
-; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
-; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
-; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
-; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
-; CHECK-INTERLEAVED: [[FOR_INC]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_2]] = phi i32 [ [[SUM_1]], %[[FOR_BODY]] ], [ [[ADD]], %[[IF_THEN]] ]
-; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK-INTERLEAVED: [[EXIT]]:
-; CHECK-INTERLEAVED-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
-; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_2_LCSSA]]
+; CHECK-LABEL: define i32 @reduction_before_pred(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK: [[VECTOR_PH]]:
+; CHECK-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK: [[VECTOR_BODY]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP6]])
+; CHECK-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[PARTIAL_REDUCE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP9]], align 1
+; CHECK-NEXT: [[TMP10:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD4]], zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD5]], zeroinitializer
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP2]]
+; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i8> poison)
+; CHECK-NEXT: [[TMP14:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE7]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP15]])
+; CHECK-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD6]] to <vscale x 16 x i32>
+; CHECK-NEXT: [[TMP17:%.*]] = select <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i32> [[TMP16]], <vscale x 16 x i32> zeroinitializer
+; CHECK-NEXT: [[PARTIAL_REDUCE8]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE3]], <vscale x 16 x i32> [[TMP17]])
+; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
+; CHECK-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[SCALAR_PH]]
+; CHECK: [[SCALAR_PH]]:
;
; CHECK-TAILFOLD-LABEL: define i32 @reduction_before_pred(
; CHECK-TAILFOLD-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
>From 52d907504cfe5289cfe58c3a230e70b01f354110 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Thu, 25 Jun 2026 10:29:40 +0000
Subject: [PATCH 13/15] Make ExitValue use the blend as operand if present
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 13 ++--
.../AArch64/partial-reduce-with-predicate.ll | 72 +++++++------------
2 files changed, 34 insertions(+), 51 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 302b51900e233..71b3a5c1e4551 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6546,12 +6546,16 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
}
// Check if WidenRecipe is the final result of the reduction. If so look
- // through the select recipes introduced by tail-folding or predicated
- // reductions for the BinOp's exit value.
+ // through the select recipes introduced by tail-folding. When a Blend
+ // guards the reduction, the tail-fold select wraps the Blend,
+ // otherwise it wraps the WidenRecipe directly.
+ VPValue *ExitSearch =
+ Chain.Blend ? cast<VPValue>(Chain.Blend) : cast<VPValue>(WidenRecipe);
+
VPValue *TailFoldCond = nullptr;
auto *ExitValue = cast_or_null<VPInstruction>(findUserOf(
- WidenRecipe, m_Select(m_VPValue(TailFoldCond), m_Specific(WidenRecipe),
- m_Specific(RdxPhi))));
+ ExitSearch, m_Select(m_VPValue(TailFoldCond), m_Specific(ExitSearch),
+ m_Specific(RdxPhi))));
VPValue *BlendCond = Chain.Blend ? Chain.Blend->getMask(0) : nullptr;
VPValue *Cond = TailFoldCond;
@@ -6948,7 +6952,6 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
return PhiR == RedPhiR;
-
auto *R = cast<VPSingleDefRecipe>(U);
if (auto *Blend = dyn_cast<VPBlendRecipe>(R))
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index 28aec1c2e0ed5..645264a81403d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -61,7 +61,7 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD: [[VECTOR_BODY]]:
; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
@@ -70,15 +70,14 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
; CHECK-TAILFOLD-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
; CHECK-TAILFOLD-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
-; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
-; CHECK-TAILFOLD-NEXT: [[TMP8]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
; CHECK-TAILFOLD-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
-; CHECK-TAILFOLD-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP8]])
+; CHECK-TAILFOLD-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP11]]
@@ -167,7 +166,7 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD: [[VECTOR_BODY]]:
; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
@@ -176,15 +175,14 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
; CHECK-TAILFOLD-NEXT: [[TMP6:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
; CHECK-TAILFOLD-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
-; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
-; CHECK-TAILFOLD-NEXT: [[TMP8]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-NEXT: [[TMP10:%.*]] = xor i1 [[TMP9]], true
; CHECK-TAILFOLD-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
-; CHECK-TAILFOLD-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP8]])
+; CHECK-TAILFOLD-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP11]]
@@ -281,7 +279,7 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD: [[VECTOR_BODY]]:
; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
@@ -294,15 +292,14 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD2]] to <vscale x 16 x i32>
; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP7]], [[TMP8]]
; CHECK-TAILFOLD-NEXT: [[TMP10:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP9]], <vscale x 16 x i32> zeroinitializer
-; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP10]])
-; CHECK-TAILFOLD-NEXT: [[TMP11]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP10]])
; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-TAILFOLD-NEXT: [[TMP12:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
; CHECK-TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
-; CHECK-TAILFOLD-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP11]])
+; CHECK-TAILFOLD-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP14]]
@@ -404,7 +401,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD: [[VECTOR_BODY]]:
; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD]], zeroinitializer
@@ -417,15 +414,15 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-TAILFOLD-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD2]] to <vscale x 16 x i32>
; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP7]], [[TMP8]]
; CHECK-TAILFOLD-NEXT: [[TMP10:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP9]], <vscale x 16 x i32> zeroinitializer
-; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP10]])
-; CHECK-TAILFOLD-NEXT: [[TMP11]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP10]])
; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-TAILFOLD-NEXT: [[TMP12:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-NEXT: [[TMP13:%.*]] = xor i1 [[TMP12]], true
; CHECK-TAILFOLD-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
-; CHECK-TAILFOLD-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP11]])
+; CHECK-TAILFOLD-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-TAILFOLD-NEXT: [[TMP14:%.*]] = sub i32 0, [[TMP15]]
; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP14]]
@@ -648,11 +645,11 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-TAILFOLD: [[VECTOR_BODY]]:
; CHECK-TAILFOLD-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 16 x i1> [ [[ACTIVE_LANE_MASK_ENTRY]], %[[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 16 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-TAILFOLD-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
; CHECK-TAILFOLD-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP2]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
; CHECK-TAILFOLD-NEXT: [[TMP3:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
-; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP3]])
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP3]])
; CHECK-TAILFOLD-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP4]], <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i8> poison)
; CHECK-TAILFOLD-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]], zeroinitializer
@@ -661,15 +658,14 @@ define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %
; CHECK-TAILFOLD-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
; CHECK-TAILFOLD-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD2]] to <vscale x 16 x i32>
; CHECK-TAILFOLD-NEXT: [[TMP9:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
-; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE3:%.*]] = call <vscale x 16 x i32> @llvm.vector.partial.reduce.add.nxv16i32.nxv16i32(<vscale x 16 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
-; CHECK-TAILFOLD-NEXT: [[TMP10]] = select <vscale x 16 x i1> [[ACTIVE_LANE_MASK]], <vscale x 16 x i32> [[PARTIAL_REDUCE3]], <vscale x 16 x i32> [[VEC_PHI]]
+; CHECK-TAILFOLD-NEXT: [[PARTIAL_REDUCE3]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
; CHECK-TAILFOLD-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
; CHECK-TAILFOLD-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[INDEX_NEXT]], i64 [[N]])
; CHECK-TAILFOLD-NEXT: [[TMP11:%.*]] = extractelement <vscale x 16 x i1> [[ACTIVE_LANE_MASK_NEXT]], i64 0
; CHECK-TAILFOLD-NEXT: [[TMP12:%.*]] = xor i1 [[TMP11]], true
; CHECK-TAILFOLD-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-TAILFOLD: [[MIDDLE_BLOCK]]:
-; CHECK-TAILFOLD-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv16i32(<vscale x 16 x i32> [[TMP10]])
+; CHECK-TAILFOLD-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE3]])
; CHECK-TAILFOLD-NEXT: br label %[[EXIT:.*]]
; CHECK-TAILFOLD: [[EXIT]]:
; CHECK-TAILFOLD-NEXT: ret i32 [[TMP13]]
@@ -711,13 +707,9 @@ exit:
; Negative test - expect not to generate a partial reduction as phi
; has two update values
define i32 @partial_reduce_if_else(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @partial_reduce_if_else(
-; CHECK-INTERLEAVE1-NOT: llvm.vector.partial.reduce
-; CHECK-INTERLEAVE1: call i32 @llvm.vector.reduce.add
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @partial_reduce_if_else(
-; CHECK-INTERLEAVED-NOT: llvm.vector.partial.reduce
-; CHECK-INTERLEAVED: call i32 @llvm.vector.reduce.add
+; CHECK-LABEL: define i32 @partial_reduce_if_else(
+; CHECK-NOT: llvm.vector.partial.reduce
+; CHECK: call i32 @llvm.vector.reduce.add
;
; CHECK-TAILFOLD-LABEL: define i32 @partial_reduce_if_else(
; CHECK-TAILFOLD-NOT: llvm.vector.partial.reduce
@@ -761,13 +753,9 @@ exit:
; Negative test - expect not to generate the partial reduction as
; there are more than two incomng values to the predicated phi.
define i32 @partial_reduce_three_incoming(ptr %a, ptr %b, ptr %cond_a, ptr %cond_b, i64 %N) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @partial_reduce_three_incoming(
-; CHECK-INTERLEAVE1-NOT: llvm.vector.partial.reduce
-; CHECK-INTERLEAVE1: call i32 @llvm.vector.reduce.add
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @partial_reduce_three_incoming(
-; CHECK-INTERLEAVED-NOT: llvm.vector.partial.reduce
-; CHECK-INTERLEAVED: call i32 @llvm.vector.reduce.add
+; CHECK-LABEL: define i32 @partial_reduce_three_incoming(
+; CHECK-NOT: llvm.vector.partial.reduce
+; CHECK: call i32 @llvm.vector.reduce.add
;
; CHECK-TAILFOLD-LABEL: define i32 @partial_reduce_three_incoming(
; CHECK-TAILFOLD-NOT: llvm.vector.partial.reduce
@@ -826,10 +814,6 @@ define i32 @partial_reduce_mixed_scale_factor(ptr %a8, ptr %b16,
; CHECK-INTERLEAVE1-NOT: llvm.vector.partial.reduce
; CHECK-INTERLEAVE1: call i32 @llvm.vector.reduce.add
;
-; CHECK-INTERLEAVED-LABEL: define i32 @partial_reduce_mixed_scale_factor(
-; CHECK-INTERLEAVED-NOT: llvm.vector.partial.reduce
-; CHECK-INTERLEAVED: call i32 @llvm.vector.reduce.add
-;
; CHECK-TAILFOLD-LABEL: define i32 @partial_reduce_mixed_scale_factor(
; CHECK-TAILFOLD-NOT: llvm.vector.partial.reduce
; CHECK-TAILFOLD: call i32 @llvm.vector.reduce.add
@@ -871,13 +855,9 @@ exit:
; the passthrough value is %sum, not the previous chain value %sum.1.
define i32 @partial_reduce_bad_passthrough(ptr %src, ptr noalias %src_b,
ptr %cond, i64 %N) #0 {
-; CHECK-INTERLEAVE1-LABEL: define i32 @partial_reduce_bad_passthrough(
-; CHECK-INTERLEAVE1-NOT: llvm.vector.partial.reduce
-; CHECK-INTERLEAVE1: call i32 @llvm.vector.reduce.add
-;
-; CHECK-INTERLEAVED-LABEL: define i32 @partial_reduce_bad_passthrough(
-; CHECK-INTERLEAVED-NOT: llvm.vector.partial.reduce
-; CHECK-INTERLEAVED: call i32 @llvm.vector.reduce.add
+; CHECK-LABEL: define i32 @partial_reduce_bad_passthrough(
+; CHECK-NOT: llvm.vector.partial.reduce
+; CHECK: call i32 @llvm.vector.reduce.add
;
; CHECK-TAILFOLD-LABEL: define i32 @partial_reduce_bad_passthrough(
; CHECK-TAILFOLD-NOT: llvm.vector.partial.reduce
>From d811852b951c2c0db728de52605cdf6ab16ad5aa Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Mon, 29 Jun 2026 10:37:17 +0000
Subject: [PATCH 14/15] add epilogue vec tests
---
.../partial-reduce-with-predicate-epilogue.ll | 210 ++++++++++++++++++
.../AArch64/partial-reduce-with-predicate.ll | 3 +-
2 files changed, 212 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate-epilogue.ll
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate-epilogue.ll
new file mode 100644
index 0000000000000..b29dc90cd3755
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate-epilogue.ll
@@ -0,0 +1,210 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=true -epilogue-vectorization-force-VF=4 -S < %s | FileCheck %s --check-prefixes=CHECK-EPI
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=true -epilogue-vectorization-force-VF=8 -S < %s | FileCheck %s --check-prefixes=CHECK-PARTIAL-RED-EPI
+
+target triple = "aarch64"
+
+define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
+; CHECK-EPI-LABEL: define i32 @pred_reduction(
+; CHECK-EPI-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-EPI-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-EPI-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; CHECK-EPI-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-EPI-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-EPI-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-EPI-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-EPI-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-EPI: [[VECTOR_PH]]:
+; CHECK-EPI-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-EPI-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-EPI-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-EPI: [[VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-EPI-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-EPI-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-EPI-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-EPI-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-EPI-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-EPI-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-EPI-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-EPI-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-EPI: [[MIDDLE_BLOCK]]:
+; CHECK-EPI-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-EPI-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-EPI-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-EPI-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; CHECK-EPI-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 4
+; CHECK-EPI-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
+; CHECK-EPI-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-EPI-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[VEC_PHI5:%.*]] = phi <4 x i32> [ [[TMP10]], %[[VEC_EPILOG_PH]] ], [ [[PREDPHI:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-EPI-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX4]]
+; CHECK-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP11]], align 1
+; CHECK-EPI-NEXT: [[TMP12:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD6]], zeroinitializer
+; CHECK-EPI-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-EPI-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 1 [[TMP13]], <4 x i1> [[TMP12]], <4 x i8> poison)
+; CHECK-EPI-NEXT: [[TMP14:%.*]] = zext <4 x i8> [[WIDE_MASKED_LOAD7]] to <4 x i32>
+; CHECK-EPI-NEXT: [[TMP15:%.*]] = add <4 x i32> [[VEC_PHI5]], [[TMP14]]
+; CHECK-EPI-NEXT: [[PREDPHI]] = select <4 x i1> [[TMP12]], <4 x i32> [[TMP15]], <4 x i32> [[VEC_PHI5]]
+; CHECK-EPI-NEXT: [[INDEX_NEXT8]] = add nuw i64 [[INDEX4]], 4
+; CHECK-EPI-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT8]], [[N_VEC3]]
+; CHECK-EPI-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-EPI-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PREDPHI]])
+; CHECK-EPI-NEXT: [[CMP_N9:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; CHECK-EPI-NEXT: br i1 [[CMP_N9]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-EPI: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: [[BC_MERGE_RDX10:%.*]] = phi i32 [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-EPI-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-EPI: [[FOR_BODY]]:
+; CHECK-EPI-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-EPI-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX10]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-EPI-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-EPI-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-EPI-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-EPI-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-EPI: [[IF_THEN]]:
+; CHECK-EPI-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-EPI-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-EPI-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-EPI-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-EPI-NEXT: br label %[[FOR_INC]]
+; CHECK-EPI: [[FOR_INC]]:
+; CHECK-EPI-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-EPI-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-EPI: [[EXIT]]:
+; CHECK-EPI-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-EPI-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-PARTIAL-RED-EPI-LABEL: define i32 @pred_reduction(
+; CHECK-PARTIAL-RED-EPI-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ITER_CHECK:.*]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-PARTIAL-RED-EPI-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[MIDDLE_BLOCK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_ITER_CHECK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[N_MOD_VF2:%.*]] = urem i64 [[N]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_VECTOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VEC_PHI5:%.*]] = phi <2 x i32> [ [[TMP10]], %[[VEC_EPILOG_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX4]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_LOAD6:%.*]] = load <8 x i8>, ptr [[TMP11]], align 1
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP12:%.*]] = icmp ne <8 x i8> [[WIDE_LOAD6]], zeroinitializer
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX4]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[WIDE_MASKED_LOAD7:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 [[TMP13]], <8 x i1> [[TMP12]], <8 x i8> poison)
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP14:%.*]] = zext <8 x i8> [[WIDE_MASKED_LOAD7]] to <8 x i32>
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP15:%.*]] = select <8 x i1> [[TMP12]], <8 x i32> [[TMP14]], <8 x i32> zeroinitializer
+; CHECK-PARTIAL-RED-EPI-NEXT: [[PARTIAL_REDUCE8]] = call <2 x i32> @llvm.vector.partial.reduce.add.v2i32.v8i32(<2 x i32> [[VEC_PHI5]], <8 x i32> [[TMP15]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX4]], 8
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC3]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TMP17:%.*]] = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> [[PARTIAL_REDUCE8]])
+; CHECK-PARTIAL-RED-EPI-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC3]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; CHECK-PARTIAL-RED-EPI: [[VEC_EPILOG_SCALAR_PH]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC3]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[BC_MERGE_RDX11:%.*]] = phi i32 [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-PARTIAL-RED-EPI: [[FOR_BODY]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX11]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-PARTIAL-RED-EPI-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-PARTIAL-RED-EPI: [[IF_THEN]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-PARTIAL-RED-EPI-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-PARTIAL-RED-EPI-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br label %[[FOR_INC]]
+; CHECK-PARTIAL-RED-EPI: [[FOR_INC]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-PARTIAL-RED-EPI-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-PARTIAL-RED-EPI-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-PARTIAL-RED-EPI: [[EXIT]]:
+; CHECK-PARTIAL-RED-EPI-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ [[TMP17]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ]
+; CHECK-PARTIAL-RED-EPI-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index 645264a81403d..1309a6bb268c3 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -2,7 +2,6 @@
; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK
; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -tail-folding-policy=must-fold-tail -S < %s | FileCheck %s --check-prefixes=CHECK-TAILFOLD
-target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64-none-unknown-elf"
define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
@@ -704,6 +703,8 @@ exit:
ret i32 %sum.2
}
+; UTC_ARGS: --disable
+
; Negative test - expect not to generate a partial reduction as phi
; has two update values
define i32 @partial_reduce_if_else(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
>From 26c881a00436adb87f8406e9c3228b0757ff19c4 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 30 Jun 2026 11:02:12 +0000
Subject: [PATCH 15/15] Review fixups
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 53 ++++++++-----------
1 file changed, 22 insertions(+), 31 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 71b3a5c1e4551..7860b5a012918 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6355,8 +6355,8 @@ struct VPPartialReductionChain {
/// is `1 - AccumulatorOpIdx`.
unsigned AccumulatorOpIdx;
unsigned ScaleFactor;
-
- /// Optional blend guarding the reduction.
+ /// Optional blend to represent predication for the block that updates the
+ /// reduction.
VPBlendRecipe *Blend = nullptr;
};
@@ -6545,27 +6545,23 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
ExtendedOp = NegRecipe;
}
- // Check if WidenRecipe is the final result of the reduction. If so look
- // through the select recipes introduced by tail-folding. When a Blend
- // guards the reduction, the tail-fold select wraps the Blend,
- // otherwise it wraps the WidenRecipe directly.
+ // Check if WidenRecipe is the final result of the reduction. If so, look
+ // through the Select recipe introduced by tail-folding, otherwise look
+ // through any Blend recipe introduced by predication for the block.
VPValue *ExitSearch =
Chain.Blend ? cast<VPValue>(Chain.Blend) : cast<VPValue>(WidenRecipe);
- VPValue *TailFoldCond = nullptr;
- auto *ExitValue = cast_or_null<VPInstruction>(findUserOf(
- ExitSearch, m_Select(m_VPValue(TailFoldCond), m_Specific(ExitSearch),
- m_Specific(RdxPhi))));
-
- VPValue *BlendCond = Chain.Blend ? Chain.Blend->getMask(0) : nullptr;
- VPValue *Cond = TailFoldCond;
+ VPValue *Cond = nullptr;
+ VPValue *ExitValue = cast_or_null<VPInstruction>(
+ findUserOf(ExitSearch, m_Select(m_VPValue(Cond), m_Specific(ExitSearch),
+ m_Specific(RdxPhi))));
- if (TailFoldCond && BlendCond) {
- VPBuilder Builder(WidenRecipe);
- Cond = Builder.createLogicalAnd(TailFoldCond, BlendCond,
- WidenRecipe->getDebugLoc());
- } else if (BlendCond) {
- Cond = BlendCond;
+ if (Chain.Blend) {
+ VPValue *BlendCond = Chain.Blend->getMask(0);
+ Cond = ExitValue ? VPBuilder(WidenRecipe)
+ .createLogicalAnd(Cond, BlendCond,
+ WidenRecipe->getDebugLoc())
+ : BlendCond;
}
bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
@@ -6809,16 +6805,11 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
// Work backwards from the ExitValue examining each reduction operation.
VPValue *CurrentValue = ExitValue;
while (CurrentValue != RedPhiR) {
- VPBlendRecipe *Blend = nullptr;
- if (auto *B = dyn_cast<VPBlendRecipe>(CurrentValue)) {
- if (B->getNumIncomingValues() != 2)
- return std::nullopt;
- assert(!B->isNormalized() && "Expect Blend not to be normalized.");
-
- Blend = B;
- CurrentValue = B->getIncomingValue(0);
-
- if (!CurrentValue->hasOneUse())
+ VPBlendRecipe *Blend = dyn_cast<VPBlendRecipe>(CurrentValue);
+ if (Blend) {
+ assert(!Blend->isNormalized() && "Expect Blend not to be normalized.");
+ CurrentValue = Blend->getIncomingValue(0);
+ if (Blend->getNumIncomingValues() != 2 || !CurrentValue->hasOneUse())
return std::nullopt;
}
@@ -6840,8 +6831,8 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
std::swap(Op, PrevValue);
}
- // Look for VPBlend(cond_reduce(PrevValue, Op), PrevValue), where
- // cond_reduce is equal to CurrentValue. This can be lowered as
+ // Look for VPBlend(reduce(PrevValue, Op), PrevValue), where
+ // reduce is equal to CurrentValue. This can be lowered as
// a conditional reduction by hoisting the select to the inputs.
if (Blend && Blend->getIncomingValue(1) != PrevValue)
return std::nullopt;
More information about the llvm-commits
mailing list