[llvm] [LV] Handle partial reductions with predication (PR #194859)
Jacob Crawley via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 23 03:34:42 PDT 2026
https://github.com/jacob-crawley updated https://github.com/llvm/llvm-project/pull/194859
>From 7084931ff6270b8ac09d3f40a80fb2e184df8df3 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 24 Apr 2026 14:49:09 +0000
Subject: [PATCH 1/7] [LV] Handle partial reductions with predication
Enables partial reductions to be used when a reduction is
gated by a condition inside a loop.
Previously when a loop contained a condition, this would introduce a
BLEND as the backedge of the phi, which prevented a partial
reduction from being used in a pattern where it would normally be
created.
This patch checks for a BLEND in the VPlan to determine whether
one of its operands can be transformed into a partial reduction.
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 49 +
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 4 +-
.../AArch64/partial-reduce-with-predicate.ll | 920 ++++++++++++++++++
3 files changed, 972 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index bdffe0c245a34..37041f236d9f2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6548,6 +6548,29 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
VPValue *ExitValue = cast_or_null<VPInstruction>(
findUserOf(WidenRecipe, m_Select(m_VPValue(Cond), m_Specific(WidenRecipe),
m_Specific(RdxPhi))));
+
+ // Look for a VPBlendRecipe user of the WidenRecipe. When the reduction is
+ // gated by an `if` in the loop body, the blend merges the updated value
+ // (WidenRecipe) and the unchanged phi value (RdxPhi).
+ if (!ExitValue) {
+ for (VPUser *U : WidenRecipe->users()) {
+ auto *Blend = dyn_cast<VPBlendRecipe>(cast<VPRecipeBase>(U));
+ if (!Blend || Blend->getNumIncomingValues() != 2)
+ continue;
+ VPValue *V0 = Blend->getIncomingValue(0);
+ VPValue *V1 = Blend->getIncomingValue(1);
+
+ if ((V0 == WidenRecipe) && (V1 == RdxPhi) ||
+ (V1 == WidenRecipe) && (V0 == RdxPhi)) {
+ // Use the mask of the WidenRecipe incoming as the reduction condition.
+ unsigned maskIdx = (V0 == WidenRecipe) ? 0 : 1;
+ Cond = Blend->getMask(maskIdx);
+ ExitValue = Blend;
+ break;
+ }
+ }
+ }
+
bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
RdxPhi->getBackedgeValue() == ExitValue;
assert((!ExitValue || IsLastInChain) &&
@@ -6778,6 +6801,19 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
VPValue *ExitValue = RdxResult->getOperand(0);
match(ExitValue, m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
+ // Check for a reduction in the operand of a Blend.
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(ExitValue)) {
+ if(Blend->getNumIncomingValues() == 2) {
+ VPValue *V0 = Blend->getIncomingValue(0);
+ VPValue *V1 = Blend->getIncomingValue(1);
+
+ if (V0 == RedPhiR)
+ ExitValue = V1;
+ if (V1 == RedPhiR)
+ ExitValue = V0;
+ }
+ }
+
SmallVector<VPPartialReductionChain> Chain;
RecurKind RK = RedPhiR->getRecurrenceKind();
Type *PhiType = RedPhiR->getScalarType();
@@ -6906,6 +6942,19 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
return PhiR == RedPhiR;
+
+ // Predicated reductions introduced by an 'if' inside the loop
+ // will be represented by a VPBlendRecipe with two incoming values,
+ // the reduction PHI and the reduction Bin Op.
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(U)) {
+ if (Blend->getNumIncomingValues() != 2)
+ return false;
+
+ VPValue *V0 = Blend->getIncomingValue(0);
+ VPValue *V1 = Blend->getIncomingValue(1);
+ return (V0 == Chain.ReductionBinOp && V1 == RedPhiR) || (V1 == Chain.ReductionBinOp && V0 == RedPhiR);
+ }
+
auto *R = cast<VPSingleDefRecipe>(U);
return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
match(R, m_ComputeReductionResult(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index f477faea58d1a..a80b9e1326798 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -848,9 +848,11 @@ VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
findUserOf<VPInstruction::ComputeReductionResult>(BackedgeVal))
return Res;
- // Look through selects inserted for tail folding or predicated reductions.
+ // Look through selects inserted for tail folding or predicated reductions.
VPRecipeBase *SelR =
findUserOf(BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
+ if (!SelR)
+ SelR = dyn_cast_or_null<VPBlendRecipe>(BackedgeVal->getDefiningRecipe());
if (!SelR)
return nullptr;
return findUserOf<VPInstruction::ComputeReductionResult>(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
new file mode 100644
index 0000000000000..7d53745b693d4
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -0,0 +1,920 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVE1
+; RUN: opt -passes=loop-vectorize -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-INTERLEAVED
+; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -vectorizer-maximize-bandwidth -enable-epilogue-vectorization=false -S < %s | FileCheck %s --check-prefixes=CHECK-MAXBW
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-none-unknown-elf"
+
+define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @pred_reduction(
+; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT: [[FOR_BODY_PREHEADER:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-INTERLEAVE1: [[FOR_COND_CLEANUP]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM_08:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDVARS_IV]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP11]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[TMP12]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_08]], [[CONV]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_08]], %[[FOR_COND_CLEANUP]] ]
+; CHECK-INTERLEAVE1-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVE1: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction(
+; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT: [[FOR_BODY_PREHEADER:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP11]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP12]])
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP13]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP14]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-INTERLEAVED: [[FOR_COND_CLEANUP]]:
+; CHECK-INTERLEAVED-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM_08:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDVARS_IV]]
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP17]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[TMP18]] to i32
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_08]], [[CONV]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_08]], %[[FOR_COND_CLEANUP]] ]
+; CHECK-INTERLEAVED-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVED: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @pred_reduction(
+; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-MAXBW-NEXT: [[FOR_BODY_PREHEADER:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[FOR_COND_CLEANUP_LOOPEXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[FOR_BODY_PREHEADER]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_COND_CLEANUP:.*]]
+; CHECK-MAXBW: [[FOR_COND_CLEANUP]]:
+; CHECK-MAXBW-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM_08:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDVARS_IV]]
+; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[TMP11]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[INDVARS_IV]]
+; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[CONV:%.*]] = zext i8 [[TMP12]] to i32
+; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_08]], [[CONV]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_08]], %[[FOR_COND_CLEANUP]] ]
+; CHECK-MAXBW-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[FOR_COND_CLEANUP_LOOPEXIT]], label %[[FOR_COND_CLEANUP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-MAXBW: [[FOR_COND_CLEANUP_LOOPEXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @pred_reduction_sext(
+; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = sext i8 [[VAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction_sext(
+; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE4:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP11]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP12]])
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP13]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP14]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE4]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = sext i8 [[VAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP16]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @pred_reduction_sext(
+; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = sext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP7]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP8]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-MAXBW: [[FOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[CONV:%.*]] = sext i8 [[VAL]] to i32
+; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK-MAXBW: [[EXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = sext i8 %val to i32
+ %add = add nsw i32 %sum, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VA:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VB:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV_A:%.*]] = zext i8 [[VA]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[CONV_B:%.*]] = zext i8 [[VB]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV_A]], [[CONV_B]]
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP11]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP15]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP16]])
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP17]], [[TMP18]]
+; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP19]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP20]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP22]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VA:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VB:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV_A:%.*]] = zext i8 [[VA]] to i32
+; CHECK-INTERLEAVED-NEXT: [[CONV_B:%.*]] = zext i8 [[VB]] to i32
+; CHECK-INTERLEAVED-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV_A]], [[CONV_B]]
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP22]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @pred_reduction_dotprod(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP13]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-MAXBW: [[FOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[VA:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[VB:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-MAXBW-NEXT: [[CONV_A:%.*]] = zext i8 [[VA]] to i32
+; CHECK-MAXBW-NEXT: [[CONV_B:%.*]] = zext i8 [[VB]] to i32
+; CHECK-MAXBW-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[CONV_A]], [[CONV_B]]
+; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[MUL]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK-MAXBW: [[EXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %arrayidx2, align 1
+ %arrayidx4 = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %arrayidx4, align 1
+ %ext.a = zext i8 %load.a to i32
+ %ext.b = zext i8 %load.b to i32
+ %mul = mul nuw nsw i32 %ext.a, %ext.b
+ %add = add nsw i32 %sum, %mul
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @pred_sub_reduction(
+; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP14:%.*]] = sub i32 0, [[TMP13]]
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
+; CHECK-INTERLEAVE1-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM]], [[MUL]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1]] = phi i32 [ [[SUB]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @pred_sub_reduction(
+; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE6:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP5]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP6]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP10]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[TMP11]], i64 [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD4:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP11]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD5:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD4]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP13]], [[TMP14]]
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP15]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP16]])
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD5]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP17]], [[TMP18]]
+; CHECK-INTERLEAVED-NEXT: [[TMP20:%.*]] = select <vscale x 16 x i1> [[TMP8]], <vscale x 16 x i32> [[TMP19]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE6]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP20]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]]
+; CHECK-INTERLEAVED-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE6]], [[PARTIAL_REDUCE]]
+; CHECK-INTERLEAVED-NEXT: [[TMP22:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[TMP23:%.*]] = sub i32 0, [[TMP22]]
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP23]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-INTERLEAVED-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
+; CHECK-INTERLEAVED-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
+; CHECK-INTERLEAVED-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM]], [[MUL]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1]] = phi i32 [ [[SUB]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP23]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @pred_sub_reduction(
+; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP6]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD1]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = mul nuw nsw <vscale x 16 x i32> [[TMP8]], [[TMP9]]
+; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP5]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-MAXBW-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]])
+; CHECK-MAXBW-NEXT: [[TMP14:%.*]] = sub i32 0, [[TMP13]]
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-MAXBW: [[FOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_1:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[LOAD_A:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[LOAD_B:%.*]] = load i8, ptr [[ARRAYIDX4]], align 1
+; CHECK-MAXBW-NEXT: [[EXT_A:%.*]] = zext i8 [[LOAD_A]] to i32
+; CHECK-MAXBW-NEXT: [[EXT_B:%.*]] = zext i8 [[LOAD_B]] to i32
+; CHECK-MAXBW-NEXT: [[MUL:%.*]] = mul nuw nsw i32 [[EXT_A]], [[EXT_B]]
+; CHECK-MAXBW-NEXT: [[SUB:%.*]] = sub nsw i32 [[SUM]], [[MUL]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_1]] = phi i32 [ [[SUB]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; CHECK-MAXBW: [[EXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_1_LCSSA:%.*]] = phi i32 [ [[SUM_1]], %[[FOR_INC]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_1_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.1, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %a, i64 %iv
+ %load.a = load i8, ptr %arrayidx2, align 1
+ %arrayidx4 = getelementptr inbounds nuw i8, ptr %b, i64 %iv
+ %load.b = load i8, ptr %arrayidx4, align 1
+ %ext.a = zext i8 %load.a to i32
+ %ext.b = zext i8 %load.b to i32
+ %mul = mul nuw nsw i32 %ext.a, %ext.b
+ %sub = sub nsw i32 %sum, %mul
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %sub, %if.then ], [ %sum, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.1
+}
+
+attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
>From dbb2634bfa5e011d125a6379dbeb12555d60c582 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 17 Jun 2026 10:45:27 +0000
Subject: [PATCH 2/7] Use m_SelectLike matcher to check for blend recipes
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 56 ++++++-------------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 12 ++--
.../AArch64/partial-reduce-with-predicate.ll | 36 ++++--------
3 files changed, 37 insertions(+), 67 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 37041f236d9f2..ea3390b622c83 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6543,33 +6543,21 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
}
// Check if WidenRecipe is the final result of the reduction. If so look
- // through selects for predicated reductions.
+ // through select-like recipes for predicated reductions.
VPValue *Cond = nullptr;
- VPValue *ExitValue = cast_or_null<VPInstruction>(
- findUserOf(WidenRecipe, m_Select(m_VPValue(Cond), m_Specific(WidenRecipe),
- m_Specific(RdxPhi))));
-
- // Look for a VPBlendRecipe user of the WidenRecipe. When the reduction is
- // gated by an `if` in the loop body, the blend merges the updated value
- // (WidenRecipe) and the unchanged phi value (RdxPhi).
- if (!ExitValue) {
- for (VPUser *U : WidenRecipe->users()) {
- auto *Blend = dyn_cast<VPBlendRecipe>(cast<VPRecipeBase>(U));
- if (!Blend || Blend->getNumIncomingValues() != 2)
- continue;
- VPValue *V0 = Blend->getIncomingValue(0);
- VPValue *V1 = Blend->getIncomingValue(1);
-
- if ((V0 == WidenRecipe) && (V1 == RdxPhi) ||
- (V1 == WidenRecipe) && (V0 == RdxPhi)) {
- // Use the mask of the WidenRecipe incoming as the reduction condition.
- unsigned maskIdx = (V0 == WidenRecipe) ? 0 : 1;
- Cond = Blend->getMask(maskIdx);
- ExitValue = Blend;
- break;
- }
+ VPRecipeBase *ExitRecipe = findUserOf(
+ WidenRecipe, m_SelectLike(m_VPValue(Cond), m_Specific(WidenRecipe),
+ m_Specific(RdxPhi)));
+ if (!ExitRecipe) {
+ ExitRecipe = findUserOf(WidenRecipe,
+ m_SelectLike(m_VPValue(Cond), m_Specific(RdxPhi),
+ m_Specific(WidenRecipe)));
+ if (ExitRecipe) {
+ VPBuilder Builder(WidenRecipe);
+ Cond = Builder.createNot(Cond, ExitRecipe->getDebugLoc());
}
}
+ VPValue *ExitValue = cast_or_null<VPSingleDefRecipe>(ExitRecipe);
bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
RdxPhi->getBackedgeValue() == ExitValue;
@@ -6799,20 +6787,12 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
if (!RdxResult)
return std::nullopt;
VPValue *ExitValue = RdxResult->getOperand(0);
- match(ExitValue, m_Select(m_VPValue(), m_VPValue(ExitValue), m_VPValue()));
-
- // Check for a reduction in the operand of a Blend.
- if (auto *Blend = dyn_cast<VPBlendRecipe>(ExitValue)) {
- if(Blend->getNumIncomingValues() == 2) {
- VPValue *V0 = Blend->getIncomingValue(0);
- VPValue *V1 = Blend->getIncomingValue(1);
-
- if (V0 == RedPhiR)
- ExitValue = V1;
- if (V1 == RedPhiR)
- ExitValue = V0;
- }
- }
+ VPValue *SelectedValue = nullptr;
+ if (match(ExitValue, m_SelectLike(m_VPValue(), m_VPValue(SelectedValue),
+ m_Specific(RedPhiR))) ||
+ match(ExitValue, m_SelectLike(m_VPValue(), m_Specific(RedPhiR),
+ m_VPValue(SelectedValue))))
+ ExitValue = SelectedValue;
SmallVector<VPPartialReductionChain> Chain;
RecurKind RK = RedPhiR->getRecurrenceKind();
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index a80b9e1326798..ed88dca2e732c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -848,11 +848,13 @@ VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
findUserOf<VPInstruction::ComputeReductionResult>(BackedgeVal))
return Res;
- // Look through selects inserted for tail folding or predicated reductions.
- VPRecipeBase *SelR =
- findUserOf(BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
- if (!SelR)
- SelR = dyn_cast_or_null<VPBlendRecipe>(BackedgeVal->getDefiningRecipe());
+ // Look through select-like recipes inserted for tail folding or predicated
+ // reductions.
+ VPRecipeBase *SelR = findUserOf(
+ BackedgeVal, m_SelectLike(m_VPValue(), m_VPValue(), m_VPValue()));
+ if (!SelR &&
+ match(BackedgeVal, m_SelectLike(m_VPValue(), m_VPValue(), m_VPValue())))
+ SelR = BackedgeVal->getDefiningRecipe();
if (!SelR)
return nullptr;
return findUserOf<VPInstruction::ComputeReductionResult>(
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index 7d53745b693d4..0af873aef91c5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -15,8 +15,7 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -72,8 +71,7 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
@@ -140,8 +138,7 @@ define i32 @pred_reduction(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -226,8 +223,7 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -283,8 +279,7 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
@@ -351,8 +346,7 @@ define i32 @pred_reduction_sext(ptr %src, ptr %cond, i64 %N) #0 {
; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -437,8 +431,7 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -502,8 +495,7 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
@@ -582,8 +574,7 @@ define i32 @pred_reduction_dotprod(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -680,8 +671,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
-; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
@@ -746,8 +736,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-INTERLEAVED: [[VECTOR_PH]]:
-; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 1
; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
@@ -827,8 +816,7 @@ define i32 @pred_sub_reduction(ptr %a, ptr %b, ptr %cond, i64 %N) #0 {
; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
; CHECK-MAXBW: [[VECTOR_PH]]:
-; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 4
; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
>From f48d0430e812330d60d788fd40a9154c96fa211b Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Wed, 17 Jun 2026 13:50:08 +0000
Subject: [PATCH 3/7] formatting
---
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index ea3390b622c83..7eb01626a0d0b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6932,7 +6932,8 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
VPValue *V0 = Blend->getIncomingValue(0);
VPValue *V1 = Blend->getIncomingValue(1);
- return (V0 == Chain.ReductionBinOp && V1 == RedPhiR) || (V1 == Chain.ReductionBinOp && V0 == RedPhiR);
+ return (V0 == Chain.ReductionBinOp && V1 == RedPhiR) ||
+ (V1 == Chain.ReductionBinOp && V0 == RedPhiR);
}
auto *R = cast<VPSingleDefRecipe>(U);
>From ea7e7af9023b0c93c7e7d0310889a51706d7358a Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 19 Jun 2026 11:26:31 +0000
Subject: [PATCH 4/7] Add BlendMask for tail-folding and chained reductions
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 129 ++++++++++++++++--
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 10 +-
2 files changed, 120 insertions(+), 19 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7eb01626a0d0b..2c598e1aab24c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6355,6 +6355,15 @@ struct VPPartialReductionChain {
/// is `1 - AccumulatorOpIdx`.
unsigned AccumulatorOpIdx;
unsigned ScaleFactor;
+
+ /// Mask from a VPBlendRecipe that guards the reduction (e.g. an `if` inside
+ /// the loop body).
+ VPValue *BlendMask = nullptr;
+
+ /// If true, negate the BlendMask before passing to the partial reduction.
+ /// This is needed for normalized blends where ReductionBinOp is the first
+ /// incoming value and selected when BlendMask is false.
+ bool NegateBlendMask = false;
};
static VPSingleDefRecipe *
@@ -6544,23 +6553,60 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
// Check if WidenRecipe is the final result of the reduction. If so look
// through select-like recipes for predicated reductions.
- VPValue *Cond = nullptr;
+ //
+ // There are up to two sources of predication:
+ // 1) Chain.Mask: captured from a VPBlendRecipe that gates this link with
+ // an inner `if` mask. May need negation (see Chain.NegateMask).
+ // 2) A latch select: introduced by tail-folding and/or
+ // non-predicated reductions to select between the updated and previous
+ // reduction values.
+ // Both must be combined (logical-and) and absorbed into the partial
+ // reduction's Cond, and both the blend and the latch select must be
+ // replaced by the partial reduction.
+ VPValue *Cond = Chain.BlendMask;
+ if (Cond && Chain.NegateBlendMask) {
+ VPBuilder Builder(WidenRecipe);
+ Cond = Builder.createNot(Cond, WidenRecipe->getDebugLoc());
+ }
+
+ // If the chain saw a blend, the latch select uses the
+ // blend rather than the WidenRecipe directly.
+ VPValue *SearchVal = WidenRecipe;
+ VPBlendRecipe *BlendUser = nullptr;
+ if (Chain.BlendMask) {
+ BlendUser = findUserOf<VPBlendRecipe>(WidenRecipe);
+ if (BlendUser)
+ SearchVal = BlendUser;
+ }
+
+ VPValue *LatchCond = nullptr;
VPRecipeBase *ExitRecipe = findUserOf(
- WidenRecipe, m_SelectLike(m_VPValue(Cond), m_Specific(WidenRecipe),
- m_Specific(RdxPhi)));
+ SearchVal, m_SelectLike(m_VPValue(LatchCond), m_Specific(SearchVal),
+ m_Specific(RdxPhi)));
if (!ExitRecipe) {
- ExitRecipe = findUserOf(WidenRecipe,
- m_SelectLike(m_VPValue(Cond), m_Specific(RdxPhi),
- m_Specific(WidenRecipe)));
+ ExitRecipe = findUserOf(SearchVal, m_SelectLike(m_VPValue(LatchCond),
+ m_Specific(RdxPhi),
+ m_Specific(SearchVal)));
if (ExitRecipe) {
VPBuilder Builder(WidenRecipe);
- Cond = Builder.createNot(Cond, ExitRecipe->getDebugLoc());
+ LatchCond = Builder.createNot(LatchCond, ExitRecipe->getDebugLoc());
}
}
+
+ // Combine the blend mask and the latch select mask.
+ if (LatchCond && Cond) {
+ VPBuilder Builder(WidenRecipe);
+ Cond =
+ Builder.createLogicalAnd(LatchCond, Cond, WidenRecipe->getDebugLoc());
+ } else if (LatchCond) {
+ Cond = LatchCond;
+ }
+
VPValue *ExitValue = cast_or_null<VPSingleDefRecipe>(ExitRecipe);
bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
- RdxPhi->getBackedgeValue() == ExitValue;
+ RdxPhi->getBackedgeValue() == ExitValue ||
+ (BlendUser && RdxPhi->getBackedgeValue() == BlendUser);
assert((!ExitValue || IsLastInChain) &&
"if we found ExitValue, it must match RdxPhi's backedge value");
@@ -6575,8 +6621,12 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
RdxUnordered{/*VFScaleFactor=*/Chain.ScaleFactor});
PartialRed->insertBefore(WidenRecipe);
- if (Cond)
+ // Replace uses of the latch select (if found) with PartialRed; otherwise
+ // the blend directly produces the reduction backedge value.
+ if (ExitValue)
ExitValue->replaceAllUsesWith(PartialRed);
+ if (BlendUser)
+ BlendUser->replaceAllUsesWith(PartialRed);
WidenRecipe->replaceAllUsesWith(PartialRed);
// For cost-model purposes, fold this into a VPExpression.
@@ -6802,6 +6852,44 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
// Work backwards from the ExitValue examining each reduction operation.
VPValue *CurrentValue = ExitValue;
while (CurrentValue != RedPhiR) {
+ VPValue *LinkMask = nullptr;
+ bool LinkNegateMask = false;
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentValue)) {
+ // Look through an incoming blend. We handle both:
+ // * Normalized blends ([I0, I1, M1]): introduced by inner `if`.
+ // * Non-normalized blends ([I0, M0, I1, M1]): introduced by an inner
+ // `if` combined with tail-folding (M0 and M1 already include the
+ // header / lane mask).
+ if (Blend->getNumIncomingValues() != 2)
+ return std::nullopt;
+
+ VPValue *V0 = Blend->getIncomingValue(0);
+ VPValue *V1 = Blend->getIncomingValue(1);
+ auto IsBinOpWiden = [](VPValue *V) {
+ auto *W = dyn_cast_or_null<VPWidenRecipe>(V);
+ return W && Instruction::isBinaryOp(W->getOpcode());
+ };
+
+ // Identify which incoming is the BinOp.
+ if (IsBinOpWiden(V1)) {
+ // Both forms: V1 is selected when M1 is true.
+ CurrentValue = V1;
+ LinkMask = Blend->getMask(1);
+ } else if (IsBinOpWiden(V0)) {
+ CurrentValue = V0;
+ if (Blend->isNormalized()) {
+ // Normalized [I0, I1, M1]: V0 is default, fires when M1 is false.
+ LinkMask = Blend->getMask(1);
+ LinkNegateMask = true;
+ } else {
+ // Non-normalized [I0, M0, I1, M1]: V0 fires when M0 is true.
+ LinkMask = Blend->getMask(0);
+ }
+ } else {
+ return std::nullopt;
+ }
+ }
+
auto *UpdateR = dyn_cast<VPWidenRecipe>(CurrentValue);
if (!UpdateR || !Instruction::isBinaryOp(UpdateR->getOpcode()))
return std::nullopt;
@@ -6831,7 +6919,8 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
VPPartialReductionChain Link(
{UpdateR, *ExtendedOp, RK,
PrevValue == UpdateR->getOperand(0) ? 0U : 1U,
- static_cast<unsigned>(PHISize.getKnownScalarFactor(ExtSrcSize))});
+ static_cast<unsigned>(PHISize.getKnownScalarFactor(ExtSrcSize)),
+ LinkMask, LinkNegateMask});
Chain.push_back(Link);
CurrentValue = PrevValue;
}
@@ -6932,8 +7021,24 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
VPValue *V0 = Blend->getIncomingValue(0);
VPValue *V1 = Blend->getIncomingValue(1);
- return (V0 == Chain.ReductionBinOp && V1 == RedPhiR) ||
- (V1 == Chain.ReductionBinOp && V0 == RedPhiR);
+ if (!((V0 == Chain.ReductionBinOp && V1 == RedPhiR) ||
+ (V1 == Chain.ReductionBinOp && V0 == RedPhiR)))
+ return false;
+
+ // The blend's users must be the reduction PHI's backedge, the
+ // compute-reduction-result, the latch select introduced
+ // by tail-folding, or another scaled-reduction with the same
+ // scale factor for chained reductions.
+ return all_of(Blend->users(), [&](VPUser *BU) {
+ if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(BU))
+ return PhiR == RedPhiR;
+ auto *R = dyn_cast<VPSingleDefRecipe>(BU);
+ return R &&
+ (Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
+ match(R, m_ComputeReductionResult(m_Specific(Blend))) ||
+ match(R, m_Select(m_VPValue(), m_Specific(Blend),
+ m_Specific(RedPhiR))));
+ });
}
auto *R = cast<VPSingleDefRecipe>(U);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index ed88dca2e732c..f477faea58d1a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -848,13 +848,9 @@ VPInstruction *vputils::findComputeReductionResult(VPReductionPHIRecipe *PhiR) {
findUserOf<VPInstruction::ComputeReductionResult>(BackedgeVal))
return Res;
- // Look through select-like recipes inserted for tail folding or predicated
- // reductions.
- VPRecipeBase *SelR = findUserOf(
- BackedgeVal, m_SelectLike(m_VPValue(), m_VPValue(), m_VPValue()));
- if (!SelR &&
- match(BackedgeVal, m_SelectLike(m_VPValue(), m_VPValue(), m_VPValue())))
- SelR = BackedgeVal->getDefiningRecipe();
+ // Look through selects inserted for tail folding or predicated reductions.
+ VPRecipeBase *SelR =
+ findUserOf(BackedgeVal, m_Select(m_VPValue(), m_VPValue(), m_VPValue()));
if (!SelR)
return nullptr;
return findUserOf<VPInstruction::ComputeReductionResult>(
>From cb3a700e2b2167c5fc664a971a92e74789f63c79 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Fri, 19 Jun 2026 13:44:30 +0000
Subject: [PATCH 5/7] added chained reduction test
---
.../AArch64/partial-reduce-with-predicate.ll | 241 ++++++++++++++++++
1 file changed, 241 insertions(+)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index 0af873aef91c5..e22f0e4899f45 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -905,4 +905,245 @@ exit:
ret i32 %sum.1
}
+define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @chained_pred_reduction(
+; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1:%.*]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @chained_pred_reduction(
+; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD2]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP8]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP9]], <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP10]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP11]])
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD3]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = select <vscale x 16 x i1> [[TMP7]], <vscale x 16 x i32> [[TMP12]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE4:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP13]])
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP14]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP14]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD6:%.*]] = load <vscale x 16 x i8>, ptr [[TMP15]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD5]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP16]])
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD6]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE8]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE4]], <vscale x 16 x i32> [[TMP17]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
+; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_1:%.*]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-INTERLEAVED-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @chained_pred_reduction(
+; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP5]], <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = select <vscale x 16 x i1> [[TMP4]], <vscale x 16 x i32> [[TMP6]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD1]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-MAXBW: [[FOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM]], [[CONV]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_1:%.*]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM]], %[[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-MAXBW-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-MAXBW-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
+; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-MAXBW: [[EXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ]
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
+ ; Chain to result of blend:
+ %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv
+ %bval = load i8, ptr %b.gep, align 1
+ %bconv = zext i8 %bval to i32
+ %sum.2 = add nsw i32 %sum.1, %bconv
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.2
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
>From 66d73e42b86b9f7483679e299c54dd5e41fc0074 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Mon, 22 Jun 2026 16:18:39 +0000
Subject: [PATCH 6/7] Store Blend in PartialReductionChain instead of mask
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 165 ++++--------------
1 file changed, 34 insertions(+), 131 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 2c598e1aab24c..7e992483d1933 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6356,14 +6356,8 @@ struct VPPartialReductionChain {
unsigned AccumulatorOpIdx;
unsigned ScaleFactor;
- /// Mask from a VPBlendRecipe that guards the reduction (e.g. an `if` inside
- /// the loop body).
- VPValue *BlendMask = nullptr;
-
- /// If true, negate the BlendMask before passing to the partial reduction.
- /// This is needed for normalized blends where ReductionBinOp is the first
- /// incoming value and selected when BlendMask is false.
- bool NegateBlendMask = false;
+ /// Optional blend guarding the reduction.
+ VPBlendRecipe *Blend = nullptr;
};
static VPSingleDefRecipe *
@@ -6552,61 +6546,28 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
}
// Check if WidenRecipe is the final result of the reduction. If so look
- // through select-like recipes for predicated reductions.
- //
- // There are up to two sources of predication:
- // 1) Chain.Mask: captured from a VPBlendRecipe that gates this link with
- // an inner `if` mask. May need negation (see Chain.NegateMask).
- // 2) A latch select: introduced by tail-folding and/or
- // non-predicated reductions to select between the updated and previous
- // reduction values.
- // Both must be combined (logical-and) and absorbed into the partial
- // reduction's Cond, and both the blend and the latch select must be
- // replaced by the partial reduction.
- VPValue *Cond = Chain.BlendMask;
- if (Cond && Chain.NegateBlendMask) {
- VPBuilder Builder(WidenRecipe);
- Cond = Builder.createNot(Cond, WidenRecipe->getDebugLoc());
- }
-
- // If the chain saw a blend, the latch select uses the
- // blend rather than the WidenRecipe directly.
- VPValue *SearchVal = WidenRecipe;
- VPBlendRecipe *BlendUser = nullptr;
- if (Chain.BlendMask) {
- BlendUser = findUserOf<VPBlendRecipe>(WidenRecipe);
- if (BlendUser)
- SearchVal = BlendUser;
- }
-
- VPValue *LatchCond = nullptr;
- VPRecipeBase *ExitRecipe = findUserOf(
- SearchVal, m_SelectLike(m_VPValue(LatchCond), m_Specific(SearchVal),
- m_Specific(RdxPhi)));
- if (!ExitRecipe) {
- ExitRecipe = findUserOf(SearchVal, m_SelectLike(m_VPValue(LatchCond),
- m_Specific(RdxPhi),
- m_Specific(SearchVal)));
- if (ExitRecipe) {
- VPBuilder Builder(WidenRecipe);
- LatchCond = Builder.createNot(LatchCond, ExitRecipe->getDebugLoc());
- }
- }
+ // through the select recipes introduced by tail-folding or predicated
+ // reductions for the BinOp's exit value.
+ VPValue *TailFoldCond = nullptr;
+ auto *ExitValue = cast_or_null<VPInstruction>(findUserOf(
+ WidenRecipe, m_Select(m_VPValue(TailFoldCond), m_Specific(WidenRecipe),
+ m_Specific(RdxPhi))));
+
+ VPValue *BlendCond = Chain.Blend ? Chain.Blend->getMask(0) : nullptr;
+ VPValue *Cond = TailFoldCond;
- // Combine the blend mask and the latch select mask.
- if (LatchCond && Cond) {
+ if (TailFoldCond && BlendCond) {
VPBuilder Builder(WidenRecipe);
- Cond =
- Builder.createLogicalAnd(LatchCond, Cond, WidenRecipe->getDebugLoc());
- } else if (LatchCond) {
- Cond = LatchCond;
+ Cond = Builder.createLogicalAnd(TailFoldCond, BlendCond,
+ WidenRecipe->getDebugLoc());
+ } else if (BlendCond) {
+ Cond = BlendCond;
}
- VPValue *ExitValue = cast_or_null<VPSingleDefRecipe>(ExitRecipe);
-
- bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
- RdxPhi->getBackedgeValue() == ExitValue ||
- (BlendUser && RdxPhi->getBackedgeValue() == BlendUser);
+ bool IsLastInChain =
+ RdxPhi->getBackedgeValue() == WidenRecipe ||
+ RdxPhi->getBackedgeValue() == ExitValue ||
+ (Chain.Blend && RdxPhi->getBackedgeValue() == Chain.Blend);
assert((!ExitValue || IsLastInChain) &&
"if we found ExitValue, it must match RdxPhi's backedge value");
@@ -6621,12 +6582,12 @@ static void transformToPartialReduction(const VPPartialReductionChain &Chain,
RdxUnordered{/*VFScaleFactor=*/Chain.ScaleFactor});
PartialRed->insertBefore(WidenRecipe);
- // Replace uses of the latch select (if found) with PartialRed; otherwise
+ // Replace uses of the select (if found) with PartialRed; otherwise
// the blend directly produces the reduction backedge value.
if (ExitValue)
ExitValue->replaceAllUsesWith(PartialRed);
- if (BlendUser)
- BlendUser->replaceAllUsesWith(PartialRed);
+ if (Chain.Blend)
+ Chain.Blend->replaceAllUsesWith(PartialRed);
WidenRecipe->replaceAllUsesWith(PartialRed);
// For cost-model purposes, fold this into a VPExpression.
@@ -6837,12 +6798,6 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
if (!RdxResult)
return std::nullopt;
VPValue *ExitValue = RdxResult->getOperand(0);
- VPValue *SelectedValue = nullptr;
- if (match(ExitValue, m_SelectLike(m_VPValue(), m_VPValue(SelectedValue),
- m_Specific(RedPhiR))) ||
- match(ExitValue, m_SelectLike(m_VPValue(), m_Specific(RedPhiR),
- m_VPValue(SelectedValue))))
- ExitValue = SelectedValue;
SmallVector<VPPartialReductionChain> Chain;
RecurKind RK = RedPhiR->getRecurrenceKind();
@@ -6852,42 +6807,13 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
// Work backwards from the ExitValue examining each reduction operation.
VPValue *CurrentValue = ExitValue;
while (CurrentValue != RedPhiR) {
- VPValue *LinkMask = nullptr;
- bool LinkNegateMask = false;
- if (auto *Blend = dyn_cast<VPBlendRecipe>(CurrentValue)) {
- // Look through an incoming blend. We handle both:
- // * Normalized blends ([I0, I1, M1]): introduced by inner `if`.
- // * Non-normalized blends ([I0, M0, I1, M1]): introduced by an inner
- // `if` combined with tail-folding (M0 and M1 already include the
- // header / lane mask).
- if (Blend->getNumIncomingValues() != 2)
+ VPBlendRecipe *Blend = nullptr;
+ if (auto *B = dyn_cast<VPBlendRecipe>(CurrentValue)) {
+ if (B->getNumIncomingValues() != 2)
return std::nullopt;
- VPValue *V0 = Blend->getIncomingValue(0);
- VPValue *V1 = Blend->getIncomingValue(1);
- auto IsBinOpWiden = [](VPValue *V) {
- auto *W = dyn_cast_or_null<VPWidenRecipe>(V);
- return W && Instruction::isBinaryOp(W->getOpcode());
- };
-
- // Identify which incoming is the BinOp.
- if (IsBinOpWiden(V1)) {
- // Both forms: V1 is selected when M1 is true.
- CurrentValue = V1;
- LinkMask = Blend->getMask(1);
- } else if (IsBinOpWiden(V0)) {
- CurrentValue = V0;
- if (Blend->isNormalized()) {
- // Normalized [I0, I1, M1]: V0 is default, fires when M1 is false.
- LinkMask = Blend->getMask(1);
- LinkNegateMask = true;
- } else {
- // Non-normalized [I0, M0, I1, M1]: V0 fires when M0 is true.
- LinkMask = Blend->getMask(0);
- }
- } else {
- return std::nullopt;
- }
+ Blend = B;
+ CurrentValue = B->getIncomingValue(0);
}
auto *UpdateR = dyn_cast<VPWidenRecipe>(CurrentValue);
@@ -6897,6 +6823,9 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
VPValue *Op = UpdateR->getOperand(1);
VPValue *PrevValue = UpdateR->getOperand(0);
+ if (Blend && Blend->getIncomingValue(1) != PrevValue)
+ return std::nullopt;
+
// Find the extended operand. The other operand (PrevValue) is the next link
// in the reduction chain.
std::optional<ExtendedReductionOperand> ExtendedOp =
@@ -6920,7 +6849,7 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
{UpdateR, *ExtendedOp, RK,
PrevValue == UpdateR->getOperand(0) ? 0U : 1U,
static_cast<unsigned>(PHISize.getKnownScalarFactor(ExtSrcSize)),
- LinkMask, LinkNegateMask});
+ Blend});
Chain.push_back(Link);
CurrentValue = PrevValue;
}
@@ -7012,34 +6941,8 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
return PhiR == RedPhiR;
- // Predicated reductions introduced by an 'if' inside the loop
- // will be represented by a VPBlendRecipe with two incoming values,
- // the reduction PHI and the reduction Bin Op.
- if (auto *Blend = dyn_cast<VPBlendRecipe>(U)) {
- if (Blend->getNumIncomingValues() != 2)
- return false;
-
- VPValue *V0 = Blend->getIncomingValue(0);
- VPValue *V1 = Blend->getIncomingValue(1);
- if (!((V0 == Chain.ReductionBinOp && V1 == RedPhiR) ||
- (V1 == Chain.ReductionBinOp && V0 == RedPhiR)))
- return false;
-
- // The blend's users must be the reduction PHI's backedge, the
- // compute-reduction-result, the latch select introduced
- // by tail-folding, or another scaled-reduction with the same
- // scale factor for chained reductions.
- return all_of(Blend->users(), [&](VPUser *BU) {
- if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(BU))
- return PhiR == RedPhiR;
- auto *R = dyn_cast<VPSingleDefRecipe>(BU);
- return R &&
- (Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
- match(R, m_ComputeReductionResult(m_Specific(Blend))) ||
- match(R, m_Select(m_VPValue(), m_Specific(Blend),
- m_Specific(RedPhiR))));
- });
- }
+ if (Chain.Blend == U)
+ return true;
auto *R = cast<VPSingleDefRecipe>(U);
return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
>From b7a764a497d7c78aa2f6d8b7f09849f7fc35f043 Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley at arm.com>
Date: Tue, 23 Jun 2026 10:28:55 +0000
Subject: [PATCH 7/7] Added test with reduction before predicate
---
.../Transforms/Vectorize/VPlanTransforms.cpp | 12 +-
.../AArch64/partial-reduce-with-predicate.ll | 261 +++++++++++++++++-
2 files changed, 258 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7e992483d1933..a6ceffcb2b07e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -6823,9 +6823,6 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
VPValue *Op = UpdateR->getOperand(1);
VPValue *PrevValue = UpdateR->getOperand(0);
- if (Blend && Blend->getIncomingValue(1) != PrevValue)
- return std::nullopt;
-
// Find the extended operand. The other operand (PrevValue) is the next link
// in the reduction chain.
std::optional<ExtendedReductionOperand> ExtendedOp =
@@ -6837,6 +6834,9 @@ getScaledReductions(VPReductionPHIRecipe *RedPhiR, VPCostContext &CostCtx,
std::swap(Op, PrevValue);
}
+ if (Blend && Blend->getIncomingValue(1) != PrevValue)
+ return std::nullopt;
+
Type *ExtSrcType = ExtendedOp->ExtendA.SrcType;
TypeSize ExtSrcSize = ExtSrcType->getPrimitiveSizeInBits();
if (!PHISize.hasKnownScalarFactor(ExtSrcSize))
@@ -6941,8 +6941,10 @@ void VPlanTransforms::createPartialReductions(VPlan &Plan,
if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(U))
return PhiR == RedPhiR;
- if (Chain.Blend == U)
- return true;
+ if (auto *Blend = dyn_cast<VPBlendRecipe>(U))
+ return any_of(Chains, [Blend](const VPPartialReductionChain &C) {
+ return C.Blend == Blend;
+ });
auto *R = cast<VPSingleDefRecipe>(U);
return Chain.ScaleFactor == ScaledReductionMap.lookup_or(R, 0) ||
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
index e22f0e4899f45..5eca50cfe958b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-with-predicate.ll
@@ -907,7 +907,7 @@ exit:
define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 {
; CHECK-INTERLEAVE1-LABEL: define i32 @chained_pred_reduction(
-; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
@@ -935,7 +935,7 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -965,13 +965,13 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-INTERLEAVE1-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-INTERLEAVE1: [[EXIT]]:
; CHECK-INTERLEAVE1-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_2_LCSSA]]
;
; CHECK-INTERLEAVED-LABEL: define i32 @chained_pred_reduction(
-; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
@@ -1013,7 +1013,7 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE8]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE4]], <vscale x 16 x i32> [[TMP17]])
; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
@@ -1044,13 +1044,13 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-INTERLEAVED-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-INTERLEAVED: [[EXIT]]:
; CHECK-INTERLEAVED-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_2_LCSSA]]
;
; CHECK-MAXBW-LABEL: define i32 @chained_pred_reduction(
-; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
@@ -1078,7 +1078,7 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
-; CHECK-MAXBW-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -1108,7 +1108,7 @@ define i32 @chained_pred_reduction(ptr %src, ptr noalias %src_b, ptr %cond, i64
; CHECK-MAXBW-NEXT: [[SUM_2]] = add nsw i32 [[SUM_1]], [[BCONV]]
; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
-; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
; CHECK-MAXBW: [[EXIT]]:
; CHECK-MAXBW-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
; CHECK-MAXBW-NEXT: ret i32 [[SUM_2_LCSSA]]
@@ -1133,7 +1133,6 @@ if.then:
for.inc:
%sum.1 = phi i32 [ %add, %if.then ], [ %sum, %for.body ]
- ; Chain to result of blend:
%b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv
%bval = load i8, ptr %b.gep, align 1
%bconv = zext i8 %bval to i32
@@ -1146,4 +1145,246 @@ exit:
ret i32 %sum.2
}
+define i32 @reduction_before_pred(ptr %src, ptr noalias %src_b, ptr %cond, i64 %N) #0 {
+; CHECK-INTERLEAVE1-LABEL: define i32 @reduction_before_pred(
+; CHECK-INTERLEAVE1-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVE1-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVE1-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP4]])
+; CHECK-INTERLEAVE1-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVE1-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVE1-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVE1-NEXT: [[TMP9:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVE1-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-INTERLEAVE1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-INTERLEAVE1-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-INTERLEAVE1: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVE1-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-INTERLEAVE1-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVE1: [[SCALAR_PH]]:
+; CHECK-INTERLEAVE1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVE1: [[FOR_BODY]]:
+; CHECK-INTERLEAVE1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVE1-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVE1-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[SUM_1:%.*]] = add nsw i32 [[SUM]], [[BCONV]]
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVE1-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVE1: [[IF_THEN]]:
+; CHECK-INTERLEAVE1-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVE1-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVE1-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-INTERLEAVE1-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
+; CHECK-INTERLEAVE1-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVE1: [[FOR_INC]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_2]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_1]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVE1-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVE1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVE1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-INTERLEAVE1: [[EXIT]]:
+; CHECK-INTERLEAVE1-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVE1-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+; CHECK-INTERLEAVED-LABEL: define i32 @reduction_before_pred(
+; CHECK-INTERLEAVED-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-INTERLEAVED-NEXT: [[ENTRY:.*]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-INTERLEAVED-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 5
+; CHECK-INTERLEAVED-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
+; CHECK-INTERLEAVED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-INTERLEAVED-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[VECTOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[TMP4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP4]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP4]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD2:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP6:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP6]])
+; CHECK-INTERLEAVED-NEXT: [[TMP7:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD2]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE3:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI1]], <vscale x 16 x i32> [[TMP7]])
+; CHECK-INTERLEAVED-NEXT: [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP9:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP8]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD4:%.*]] = load <vscale x 16 x i8>, ptr [[TMP8]], align 1
+; CHECK-INTERLEAVED-NEXT: [[WIDE_LOAD5:%.*]] = load <vscale x 16 x i8>, ptr [[TMP9]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TMP10:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD4]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP11:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD5]], zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-INTERLEAVED-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP2]]
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP12]], <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[WIDE_MASKED_LOAD6:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP13]], <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i8> poison)
+; CHECK-INTERLEAVED-NEXT: [[TMP14:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP15:%.*]] = select <vscale x 16 x i1> [[TMP10]], <vscale x 16 x i32> [[TMP14]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE7]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP15]])
+; CHECK-INTERLEAVED-NEXT: [[TMP16:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD6]] to <vscale x 16 x i32>
+; CHECK-INTERLEAVED-NEXT: [[TMP17:%.*]] = select <vscale x 16 x i1> [[TMP11]], <vscale x 16 x i32> [[TMP16]], <vscale x 16 x i32> zeroinitializer
+; CHECK-INTERLEAVED-NEXT: [[PARTIAL_REDUCE8]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE3]], <vscale x 16 x i32> [[TMP17]])
+; CHECK-INTERLEAVED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
+; CHECK-INTERLEAVED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-INTERLEAVED: [[MIDDLE_BLOCK]]:
+; CHECK-INTERLEAVED-NEXT: [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[PARTIAL_REDUCE8]], [[PARTIAL_REDUCE7]]
+; CHECK-INTERLEAVED-NEXT: [[TMP19:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
+; CHECK-INTERLEAVED-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-INTERLEAVED: [[SCALAR_PH]]:
+; CHECK-INTERLEAVED-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP19]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-INTERLEAVED: [[FOR_BODY]]:
+; CHECK-INTERLEAVED-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-INTERLEAVED-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-INTERLEAVED-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-INTERLEAVED-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[SUM_1:%.*]] = add nsw i32 [[SUM]], [[BCONV]]
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-INTERLEAVED-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-INTERLEAVED-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-INTERLEAVED: [[IF_THEN]]:
+; CHECK-INTERLEAVED-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-INTERLEAVED-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-INTERLEAVED-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-INTERLEAVED-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
+; CHECK-INTERLEAVED-NEXT: br label %[[FOR_INC]]
+; CHECK-INTERLEAVED: [[FOR_INC]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_2]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_1]], %[[FOR_BODY]] ]
+; CHECK-INTERLEAVED-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-INTERLEAVED-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-INTERLEAVED-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-INTERLEAVED: [[EXIT]]:
+; CHECK-INTERLEAVED-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP19]], %[[MIDDLE_BLOCK]] ]
+; CHECK-INTERLEAVED-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+; CHECK-MAXBW-LABEL: define i32 @reduction_before_pred(
+; CHECK-MAXBW-SAME: ptr [[SRC:%.*]], ptr noalias [[SRC_B:%.*]], ptr [[COND:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-MAXBW-NEXT: [[ENTRY:.*]]:
+; CHECK-MAXBW-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-MAXBW-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
+; CHECK-MAXBW-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-MAXBW: [[VECTOR_PH]]:
+; CHECK-MAXBW-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP2]]
+; CHECK-MAXBW-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; CHECK-MAXBW-NEXT: br label %[[VECTOR_BODY:.*]]
+; CHECK-MAXBW: [[VECTOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE2:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 16 x i8>, ptr [[TMP3]], align 1
+; CHECK-MAXBW-NEXT: [[TMP4:%.*]] = zext <vscale x 16 x i8> [[WIDE_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE:%.*]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[VEC_PHI]], <vscale x 16 x i32> [[TMP4]])
+; CHECK-MAXBW-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_LOAD1:%.*]] = load <vscale x 16 x i8>, ptr [[TMP5]], align 1
+; CHECK-MAXBW-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 16 x i8> [[WIDE_LOAD1]], zeroinitializer
+; CHECK-MAXBW-NEXT: [[TMP7:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-MAXBW-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 16 x i8> @llvm.masked.load.nxv16i8.p0(ptr align 1 [[TMP7]], <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i8> poison)
+; CHECK-MAXBW-NEXT: [[TMP8:%.*]] = zext <vscale x 16 x i8> [[WIDE_MASKED_LOAD]] to <vscale x 16 x i32>
+; CHECK-MAXBW-NEXT: [[TMP9:%.*]] = select <vscale x 16 x i1> [[TMP6]], <vscale x 16 x i32> [[TMP8]], <vscale x 16 x i32> zeroinitializer
+; CHECK-MAXBW-NEXT: [[PARTIAL_REDUCE2]] = call <vscale x 4 x i32> @llvm.vector.partial.reduce.add.nxv4i32.nxv16i32(<vscale x 4 x i32> [[PARTIAL_REDUCE]], <vscale x 16 x i32> [[TMP9]])
+; CHECK-MAXBW-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-MAXBW-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-MAXBW: [[MIDDLE_BLOCK]]:
+; CHECK-MAXBW-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[PARTIAL_REDUCE2]])
+; CHECK-MAXBW-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; CHECK-MAXBW-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-MAXBW: [[SCALAR_PH]]:
+; CHECK-MAXBW-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-MAXBW-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK-MAXBW: [[FOR_BODY]]:
+; CHECK-MAXBW-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[FOR_INC:.*]] ]
+; CHECK-MAXBW-NEXT: [[SUM:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[SUM_2:%.*]], %[[FOR_INC]] ]
+; CHECK-MAXBW-NEXT: [[B_GEP:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC_B]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[BVAL:%.*]] = load i8, ptr [[B_GEP]], align 1
+; CHECK-MAXBW-NEXT: [[BCONV:%.*]] = zext i8 [[BVAL]] to i32
+; CHECK-MAXBW-NEXT: [[SUM_1:%.*]] = add nsw i32 [[SUM]], [[BCONV]]
+; CHECK-MAXBW-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[COND]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[C:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-MAXBW-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i8 [[C]], 0
+; CHECK-MAXBW-NEXT: br i1 [[TOBOOL_NOT]], label %[[FOR_INC]], label %[[IF_THEN:.*]]
+; CHECK-MAXBW: [[IF_THEN]]:
+; CHECK-MAXBW-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 [[IV]]
+; CHECK-MAXBW-NEXT: [[VAL:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1
+; CHECK-MAXBW-NEXT: [[CONV:%.*]] = zext i8 [[VAL]] to i32
+; CHECK-MAXBW-NEXT: [[ADD:%.*]] = add nsw i32 [[SUM_1]], [[CONV]]
+; CHECK-MAXBW-NEXT: br label %[[FOR_INC]]
+; CHECK-MAXBW: [[FOR_INC]]:
+; CHECK-MAXBW-NEXT: [[SUM_2]] = phi i32 [ [[ADD]], %[[IF_THEN]] ], [ [[SUM_1]], %[[FOR_BODY]] ]
+; CHECK-MAXBW-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; CHECK-MAXBW-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; CHECK-MAXBW-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
+; CHECK-MAXBW: [[EXIT]]:
+; CHECK-MAXBW-NEXT: [[SUM_2_LCSSA:%.*]] = phi i32 [ [[SUM_2]], %[[FOR_INC]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
+; CHECK-MAXBW-NEXT: ret i32 [[SUM_2_LCSSA]]
+;
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+ %sum = phi i32 [ 0, %entry ], [ %sum.2, %for.inc ]
+
+ %b.gep = getelementptr inbounds nuw i8, ptr %src_b, i64 %iv
+ %bval = load i8, ptr %b.gep, align 1
+ %bconv = zext i8 %bval to i32
+ %sum.1 = add nsw i32 %sum, %bconv
+
+ %arrayidx = getelementptr inbounds nuw i8, ptr %cond, i64 %iv
+ %c = load i8, ptr %arrayidx, align 1
+ %tobool.not = icmp eq i8 %c, 0
+ br i1 %tobool.not, label %for.inc, label %if.then
+
+if.then:
+ %arrayidx2 = getelementptr inbounds nuw i8, ptr %src, i64 %iv
+ %val = load i8, ptr %arrayidx2, align 1
+ %conv = zext i8 %val to i32
+ %add = add nsw i32 %sum.1, %conv
+ br label %for.inc
+
+for.inc:
+ %sum.2 = phi i32 [ %add, %if.then ], [ %sum.1, %for.body ]
+ %iv.next = add nuw nsw i64 %iv, 1
+ %exitcond.not = icmp eq i64 %iv.next, %N
+ br i1 %exitcond.not, label %exit, label %for.body
+
+exit:
+ ret i32 %sum.2
+}
+
attributes #0 = { vscale_range(1,16) "target-features"="+sve" }
More information about the llvm-commits
mailing list