[llvm] fb13f6f - [VPlan] Append recipes created via builder to worklist
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 03:11:43 PDT 2026
Author: Luke Lau
Date: 2026-09-25T11:11:33+01:00
New Revision: fb13f6fa52cbcd501b0dcac128254818e2d24214
URL: https://github.com/llvm/llvm-project/commit/fb13f6fa52cbcd501b0dcac128254818e2d24214
DIFF: https://github.com/llvm/llvm-project/commit/fb13f6fa52cbcd501b0dcac128254818e2d24214.diff
LOG: [VPlan] Append recipes created via builder to worklist
(#213900)
The previous PR appended the top most created recipe to the worklist,
and this PR extends it to any other nested recipes that were created,
similar to InstCombine.
This removes the header mask in a good few more places on RISC-V as
measured on SPEC CPU 2017, e.g. for the following loop:
```c
long f(const int *p, const int *q, long n) {
long a = 0, b = 0;
for (long i = 0;; i++) {
if (p[i] && q[i]) { a += i; b += i; }
if (i + 1 == n) break;
}
return a + b;
}
```
Before:
```asm
LBB0_1:
sh2add a4, a3, a0
vsetvli a5, a2, e32, m1, ta, ma
vle32.v v16, (a4)
vmsne.vi v0, v16, 0
sh2add a4, a3, a1
vsetvli zero, zero, e64, m2, ta, ma
vmsltu.vx v16, v10, a5
vle32.v v17, (a4), v0.t
vsetvli zero, zero, e32, m1, ta, ma
vmsne.vi v17, v17, 0
vmand.mm v17, v0, v17
vmand.mm v0, v16, v17
vsetvli zero, zero, e64, m2, tu, mu
vadd.vv v8, v14, v8, v0.t
vadd.vv v12, v14, v12, v0.t
vsetvli zero, zero, e64, m2, ta, ma
vadd.vx v14, v14, a5
sub a2, a2, a5
add a3, a3, a5
bnez a2, .LBB0_1
```
After:
```asm
.LBB0_1:
sh2add a4, a3, a0
vsetvli a5, a2, e32, m1, ta, ma
vle32.v v14, (a4)
vmsne.vi v0, v14, 0
sh2add a4, a3, a1
vle32.v v14, (a4), v0.t
vmsne.vi v14, v14, 0
vmand.mm v0, v0, v14
vsetvli zero, zero, e64, m2, tu, mu
vadd.vv v8, v12, v8, v0.t
vadd.vv v10, v12, v10, v0.t
vsetvli zero, zero, e64, m2, ta, ma
vadd.vx v12, v12, a5
sub a2, a2, a5
add a3, a3, a5
bnez a2, .LBB0_1
```
We can also remove the (X && Y) | (X && !Y) -> X combine, but this is
left for another NFC PR.
Added:
Modified:
llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
llvm/lib/Transforms/Vectorize/VPlan.cpp
llvm/lib/Transforms/Vectorize/VPlan.h
llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
llvm/lib/Transforms/Vectorize/VPlanTransforms.h
llvm/lib/Transforms/Vectorize/VPlanUtils.h
llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index bb83ef7dec883..465f9331ba0f3 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -25,6 +25,7 @@
#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
#include "VPlan.h"
+#include "VPlanUtils.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Support/InstructionCost.h"
@@ -97,8 +98,17 @@ void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop,
} // namespace LoopVectorizationUtils
-/// VPlan-based builder utility analogous to IRBuilder.
-class VPBuilder {
+/// Default inserter for VPBuilderBase, inserting \p R at \p It in \p VPBB.
+struct VPBuilderDefaultInserter {
+ void insertHelper(VPRecipeBase *R, VPBasicBlock *VPBB,
+ VPBasicBlock::iterator It) {
+ VPBB->insert(R, It);
+ }
+};
+
+/// VPlan-based builder utility similar to IRBuilder. Recipes are inserted via
+/// \p InserterTy.
+template <typename InserterTy> class VPBuilderBase : public InserterTy {
private:
class VPInsertPoint {
VPBasicBlock *Block = nullptr;
@@ -132,10 +142,11 @@ class VPBuilder {
VPInsertPoint InsertPt;
+protected:
/// Insert \p VPI in BB at InsertPt if BB is set.
template <typename T> T *tryInsertInstruction(T *R) {
if (InsertPt)
- InsertPt.getBlock()->insert(R, InsertPt.getIterator());
+ InserterTy::insertHelper(R, InsertPt.getBlock(), InsertPt.getIterator());
return R;
}
@@ -153,17 +164,18 @@ class VPBuilder {
return *InsertPt.getBlock()->getPlan();
}
- VPBuilder() = default;
- VPBuilder(const VPInsertPoint &IP) : InsertPt(IP) {}
- VPBuilder(VPBasicBlock *TheBB, VPBasicBlock::iterator IP)
+ VPBuilderBase() = default;
+ VPBuilderBase(const VPInsertPoint &IP) : InsertPt(IP) {}
+ VPBuilderBase(InserterTy Inserter) : InserterTy(Inserter) {}
+ VPBuilderBase(VPBasicBlock *TheBB, VPBasicBlock::iterator IP)
: InsertPt(TheBB, IP) {}
/// Get the recipe at the current insert point or nullptr if the insert point
/// is the end of the block.
VPRecipeBase *getRecipeAtInsertPoint() const { return InsertPt; }
- /// Create a VPBuilder to insert after \p R.
- static VPBuilder getToInsertAfter(VPRecipeBase *R) {
+ /// Create a builder to insert after \p R.
+ static VPBuilderBase getToInsertAfter(VPRecipeBase *R) {
return {R->getParent(), std::next(R->getIterator())};
}
@@ -182,7 +194,7 @@ class VPBuilder {
/// Insert \p R at the current insertion point. Returns \p R unchanged.
template <typename T> [[maybe_unused]] T *insert(T *R) {
- InsertPt.getBlock()->insert(R, InsertPt.getIterator());
+ InserterTy::insertHelper(R, InsertPt.getBlock(), InsertPt.getIterator());
return R;
}
@@ -341,7 +353,16 @@ class VPBuilder {
/// result, then select between \p TrueVal and \p FalseVal.
VPInstruction *createAnyOfReduction(VPValue *ChainOp, VPValue *TrueVal,
VPValue *FalseVal,
- DebugLoc DL = DebugLoc::getUnknown());
+ DebugLoc DL = DebugLoc::getUnknown()) {
+ assert(ChainOp->getScalarType()->isIntegerTy(1) &&
+ "ChainOp must be i1 for AnyOf reduction");
+ VPIRFlags Flags(RecurKind::Or, /*IsOrdered=*/false, /*IsInLoop=*/false,
+ FastMathFlags());
+ auto *OrReduce = createNaryOp(VPInstruction::ComputeReductionResult,
+ {ChainOp}, Flags, DL);
+ auto *Freeze = createNaryOp(Instruction::Freeze, {OrReduce}, DL);
+ return createSelect(Freeze, TrueVal, FalseVal, DL, "rdx.select");
+ }
VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,
DebugLoc DL = DebugLoc::getUnknown(),
@@ -519,7 +540,25 @@ class VPBuilder {
/// with element type \p SourceElementTy.
VPSingleDefRecipe *createConsecutiveVectorPointer(VPValue *Ptr,
Type *SourceElementTy,
- bool Reverse, DebugLoc DL);
+ bool Reverse, DebugLoc DL) {
+ VPlan &Plan = getPlan();
+ GEPNoWrapFlags Flags = vputils::getGEPFlagsForPtr(Ptr);
+ if (Reverse) {
+ // When folding the tail, we may compute an address that we don't in the
+ // original scalar loop: drop the GEP no-wrap flags in this case.
+ // Otherwise preserve existing flags without no-unsigned-wrap, as we will
+ // emit negative indices.
+ GEPNoWrapFlags ReverseFlags = Plan.hasTailFolded()
+ ? GEPNoWrapFlags::none()
+ : Flags.withoutNoUnsignedWrap();
+ return tryInsertInstruction(
+ new VPVectorEndPointerRecipe(Ptr, &Plan.getVF(), SourceElementTy,
+ /*Stride=*/-1, ReverseFlags, DL));
+ }
+ Type *StrideTy = Plan.getDataLayout().getIndexType(Ptr->getScalarType());
+ VPValue *StrideOne = Plan.getConstantInt(StrideTy, 1);
+ return createVectorPointer(Ptr, SourceElementTy, StrideOne, Flags, DL);
+ }
VPWidenMemIntrinsicRecipe *createWidenMemIntrinsic(
Intrinsic::ID VectorIntrinsicID, ArrayRef<VPValue *> CallArguments,
@@ -556,11 +595,11 @@ class VPBuilder {
/// RAII object that stores the current insertion point and restores it when
/// the object is destroyed.
class InsertPointGuard {
- VPBuilder &Builder;
+ VPBuilderBase &Builder;
VPInsertPoint InsertPt;
public:
- InsertPointGuard(VPBuilder &B) : Builder(B), InsertPt(B.InsertPt) {}
+ InsertPointGuard(VPBuilderBase &B) : Builder(B), InsertPt(B.InsertPt) {}
InsertPointGuard(const InsertPointGuard &) = delete;
InsertPointGuard &operator=(const InsertPointGuard &) = delete;
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 80d2704fa6139..80d98af8354d2 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1631,19 +1631,6 @@ std::string VPSlotTracker::getOrCreateName(const VPValue *V) const {
return "<badref>";
}
-VPInstruction *VPBuilder::createAnyOfReduction(VPValue *ChainOp,
- VPValue *TrueVal,
- VPValue *FalseVal, DebugLoc DL) {
- assert(ChainOp->getScalarType()->isIntegerTy(1) &&
- "ChainOp must be i1 for AnyOf reduction");
- VPIRFlags Flags(RecurKind::Or, /*IsOrdered=*/false, /*IsInLoop=*/false,
- FastMathFlags());
- auto *OrReduce =
- createNaryOp(VPInstruction::ComputeReductionResult, {ChainOp}, Flags, DL);
- auto *Freeze = createNaryOp(Instruction::Freeze, {OrReduce}, DL);
- return createSelect(Freeze, TrueVal, FalseVal, DL, "rdx.select");
-}
-
bool LoopVectorizationPlanner::getDecisionAndClampRange(
const std::function<bool(ElementCount)> &Predicate, VFRange &Range) {
assert(!Range.isEmpty() && "Trying to test an empty VF range.");
@@ -1658,27 +1645,6 @@ bool LoopVectorizationPlanner::getDecisionAndClampRange(
return PredicateAtRangeStart;
}
-VPSingleDefRecipe *
-VPBuilder::createConsecutiveVectorPointer(VPValue *Ptr, Type *SourceElementTy,
- bool Reverse, DebugLoc DL) {
- VPlan &Plan = getPlan();
- GEPNoWrapFlags Flags = vputils::getGEPFlagsForPtr(Ptr);
- if (Reverse) {
- // When folding the tail, we may compute an address that we don't in the
- // original scalar loop: drop the GEP no-wrap flags in this case. Otherwise
- // preserve existing flags without no-unsigned-wrap, as we will emit
- // negative indices.
- GEPNoWrapFlags ReverseFlags = Plan.hasTailFolded()
- ? GEPNoWrapFlags::none()
- : Flags.withoutNoUnsignedWrap();
- return tryInsertInstruction(new VPVectorEndPointerRecipe(
- Ptr, &Plan.getVF(), SourceElementTy, /*Stride=*/-1, ReverseFlags, DL));
- }
- Type *StrideTy = Plan.getDataLayout().getIndexType(Ptr->getScalarType());
- VPValue *StrideOne = Plan.getConstantInt(StrideTy, 1);
- return createVectorPointer(Ptr, SourceElementTy, StrideOne, Flags, DL);
-}
-
VPlan &LoopVectorizationPlanner::getPlanFor(ElementCount VF) const {
assert(count_if(VPlans,
[VF](const VPlanPtr &Plan) { return Plan->hasVF(VF); }) ==
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 30af5f576444e..e4778489a4060 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -62,7 +62,9 @@ class SCEV;
class SCEVPredicate;
class Type;
class VPBasicBlock;
-class VPBuilder;
+struct VPBuilderDefaultInserter;
+template <typename InserterTy = VPBuilderDefaultInserter> class VPBuilderBase;
+using VPBuilder = VPBuilderBase<>;
class VPDominatorTree;
class VPRegionBlock;
class VPlan;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 7f135c97e6392..09fdaa63bab99 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1172,9 +1172,7 @@ static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
/// Try to simplify logical and bitwise recipes in \p Def.
static VPValue *simplifyLogicalRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
// Simplify (X && Y) | (X && !Y) -> X.
- // TODO: Split up into simpler, modular combines: (X && Y) | (X && Z) into X
- // && (Y | Z) and (X | !X) into true. This requires queuing newly created
- // recipes to be visited during simplification.
+ // TODO: Remove now that we have smaller combines for this.
VPValue *X, *Y;
if (match(Def,
m_c_BinaryOr(m_LogicalAnd(m_VPValue(X), m_VPValue(Y)),
@@ -1376,8 +1374,27 @@ static bool isAvailableAtEndOf(VPValue *V, const VPBasicBlock *VPBB) {
return DefR ? DefR->getParent() == VPBB : isa<VPIRValue>(V);
}
-/// Combine \p Def into a simpler recipe. May modify or create new recipes.
-static VPSingleDefRecipe *combineRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
+namespace {
+/// Inserter for VPBuilderBase which appends all created VPSingleDefRecipes to a
+/// worklist, so they get combined as well.
+struct VPCombineInserter {
+ SmallVectorImpl<VPSingleDefRecipe *> &Worklist;
+
+ void insertHelper(VPRecipeBase *R, VPBasicBlock *VPBB,
+ VPBasicBlock::iterator It) {
+ VPBB->insert(R, It);
+ if (auto *Def = dyn_cast<VPSingleDefRecipe>(R))
+ Worklist.push_back(Def);
+ }
+};
+
+using VPCombineBuilder = VPBuilderBase<VPCombineInserter>;
+} // namespace
+
+/// Combine \p Def into a simpler recipe. May modify or create new recipes via
+/// \p Builder.
+static VPSingleDefRecipe *combineRecipe(VPlan &Plan, VPSingleDefRecipe *Def,
+ VPCombineBuilder &Builder) {
if (auto *V = simplifyRecipe(Plan, Def)) {
Def->replaceAllUsesWith(V);
return Def;
@@ -1395,12 +1412,10 @@ static VPSingleDefRecipe *combineRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
RepR->getUnderlyingInstr(), RepR->operandsWithoutMask(),
RepR->isSingleScalar(), /*Mask=*/nullptr, *RepR, *RepR,
RepR->getDebugLoc());
- Unmasked->insertBefore(RepR);
+ Builder.insert(Unmasked);
return Unmasked;
}
- VPBuilder Builder(Def);
-
// Avoid replacing VPInstructions with underlying values with new
// VPInstructions, as we would fail to create widen/replicate recpes from the
// new VPInstructions without an underlying value, and miss out on some
@@ -1716,18 +1731,19 @@ void VPlanTransforms::combineRecipes(VPlan &Plan) {
[[maybe_unused]] unsigned InitWorklistSize = Worklist.size();
+ VPCombineBuilder Builder({Worklist});
while (!Worklist.empty()) {
assert(Worklist.size() < InitWorklistSize * 2 &&
"Worklist is growing large, possible cycle?");
VPSingleDefRecipe *Def = Worklist.pop_back_val();
- VPSingleDefRecipe *New = combineRecipe(Plan, Def);
+ Builder.setInsertPoint(Def);
+ VPSingleDefRecipe *New = combineRecipe(Plan, Def, Builder);
if (!New)
continue;
if (New != Def) {
// Replace the recipe with a new one.
Def->replaceAllUsesWith(New);
Def->eraseFromParent();
- Worklist.push_back(New);
// TODO: Append users to the worklist (might need a setvector)
} else if (vputils::isDeadRecipe(*Def)) {
// Recipe was modified - it may be dead now.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 171fc3b92fe27..fd62693d6068b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -35,7 +35,6 @@ class ScalarEvolution;
class PredicatedScalarEvolution;
class TargetLibraryInfo;
class TargetTransformInfo;
-class VPBuilder;
class VPRecipeBuilder;
struct VFRange;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 1f49fb33bdfe4..7cdc846fced99 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -20,7 +20,6 @@ class MemoryLocation;
class ScalarEvolution;
class SCEV;
class PredicatedScalarEvolution;
-class VPBuilder;
} // namespace llvm
namespace llvm {
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
index 1ffe82b366a38..03a61eff45a51 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/tail-folding-complex-mask.ll
@@ -178,3 +178,164 @@ latch:
exit:
ret void
}
+
+define void @nested_mask_two_reductions(ptr %p, ptr %q, i64 %n, ptr %out) {
+; IF-EVL-LABEL: define void @nested_mask_two_reductions(
+; IF-EVL-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i64 [[N:%.*]], ptr [[OUT:%.*]]) #[[ATTR0]] {
+; IF-EVL-NEXT: [[ENTRY:.*:]]
+; IF-EVL-NEXT: br label %[[VECTOR_PH:.*]]
+; IF-EVL: [[VECTOR_PH]]:
+; IF-EVL-NEXT: [[TMP0:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; IF-EVL-NEXT: br label %[[VECTOR_BODY:.*]]
+; IF-EVL: [[VECTOR_BODY]]:
+; IF-EVL-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[CURRENT_ITERATION_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[AVL:%.*]] = phi i64 [ [[N]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IF-EVL-NEXT: [[TMP1:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true)
+; IF-EVL-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; IF-EVL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP2]], i64 0
+; IF-EVL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; IF-EVL-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
+; IF-EVL-NEXT: [[VP_OP_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr align 4 [[TMP3]], <vscale x 2 x i1> splat (i1 true), i32 [[TMP1]])
+; IF-EVL-NEXT: [[TMP4:%.*]] = icmp ne <vscale x 2 x i32> [[VP_OP_LOAD]], zeroinitializer
+; IF-EVL-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[Q]], i64 [[INDEX]]
+; IF-EVL-NEXT: [[VP_OP_LOAD2:%.*]] = call <vscale x 2 x i32> @llvm.vp.load.nxv2i32.p0(ptr align 4 [[TMP5]], <vscale x 2 x i1> [[TMP4]], i32 [[TMP1]])
+; IF-EVL-NEXT: [[TMP6:%.*]] = icmp sgt <vscale x 2 x i32> [[VP_OP_LOAD2]], zeroinitializer
+; IF-EVL-NEXT: [[TMP7:%.*]] = select <vscale x 2 x i1> [[TMP4]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> zeroinitializer
+; IF-EVL-NEXT: [[TMP9:%.*]] = add <vscale x 2 x i64> [[VEC_PHI]], [[VEC_IND]]
+; IF-EVL-NEXT: [[TMP10:%.*]] = mul <vscale x 2 x i64> [[VEC_IND]], [[VEC_IND]]
+; IF-EVL-NEXT: [[TMP11:%.*]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP10]]
+; IF-EVL-NEXT: [[TMP12]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> [[TMP7]], <vscale x 2 x i64> [[TMP9]], <vscale x 2 x i64> [[VEC_PHI]], i32 [[TMP1]])
+; IF-EVL-NEXT: [[TMP13]] = call <vscale x 2 x i64> @llvm.vp.merge.nxv2i64(<vscale x 2 x i1> [[TMP7]], <vscale x 2 x i64> [[TMP11]], <vscale x 2 x i64> [[VEC_PHI1]], i32 [[TMP1]])
+; IF-EVL-NEXT: [[CURRENT_ITERATION_NEXT]] = add i64 [[TMP2]], [[INDEX]]
+; IF-EVL-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP2]]
+; IF-EVL-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; IF-EVL-NEXT: [[TMP14:%.*]] = icmp eq i64 [[AVL_NEXT]], 0
+; IF-EVL-NEXT: br i1 [[TMP14]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; IF-EVL: [[MIDDLE_BLOCK]]:
+; IF-EVL-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP12]])
+; IF-EVL-NEXT: [[TMP16:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[TMP13]])
+; IF-EVL-NEXT: br label %[[EXIT:.*]]
+; IF-EVL: [[EXIT]]:
+; IF-EVL-NEXT: store i64 [[TMP15]], ptr [[OUT]], align 8
+; IF-EVL-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[OUT]], i64 1
+; IF-EVL-NEXT: store i64 [[TMP16]], ptr [[O1]], align 8
+; IF-EVL-NEXT: ret void
+;
+; NO-VP-LABEL: define void @nested_mask_two_reductions(
+; NO-VP-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i64 [[N:%.*]], ptr [[OUT:%.*]]) #[[ATTR0]] {
+; NO-VP-NEXT: [[ENTRY:.*]]:
+; NO-VP-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
+; NO-VP-NEXT: [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 4)
+; NO-VP-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
+; NO-VP-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; NO-VP: [[VECTOR_PH]]:
+; NO-VP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
+; NO-VP-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NO-VP-NEXT: [[TMP4:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
+; NO-VP-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP1]], i64 0
+; NO-VP-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
+; NO-VP-NEXT: br label %[[VECTOR_BODY:.*]]
+; NO-VP: [[VECTOR_BODY]]:
+; NO-VP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[VEC_IND:%.*]] = phi <vscale x 2 x i64> [ [[TMP4]], %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[VEC_PHI1:%.*]] = phi <vscale x 2 x i64> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PREDPHI2:%.*]], %[[VECTOR_BODY]] ]
+; NO-VP-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[P]], i64 [[INDEX]]
+; NO-VP-NEXT: [[WIDE_LOAD:%.*]] = load <vscale x 2 x i32>, ptr [[TMP5]], align 4
+; NO-VP-NEXT: [[TMP6:%.*]] = icmp ne <vscale x 2 x i32> [[WIDE_LOAD]], zeroinitializer
+; NO-VP-NEXT: [[TMP7:%.*]] = getelementptr i32, ptr [[Q]], i64 [[INDEX]]
+; NO-VP-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i32> @llvm.masked.load.nxv2i32.p0(ptr align 4 [[TMP7]], <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i32> poison)
+; NO-VP-NEXT: [[TMP8:%.*]] = icmp sgt <vscale x 2 x i32> [[WIDE_MASKED_LOAD]], zeroinitializer
+; NO-VP-NEXT: [[TMP9:%.*]] = select <vscale x 2 x i1> [[TMP6]], <vscale x 2 x i1> [[TMP8]], <vscale x 2 x i1> zeroinitializer
+; NO-VP-NEXT: [[TMP10:%.*]] = add <vscale x 2 x i64> [[VEC_PHI]], [[VEC_IND]]
+; NO-VP-NEXT: [[TMP11:%.*]] = mul <vscale x 2 x i64> [[VEC_IND]], [[VEC_IND]]
+; NO-VP-NEXT: [[TMP12:%.*]] = add <vscale x 2 x i64> [[VEC_PHI1]], [[TMP11]]
+; NO-VP-NEXT: [[PREDPHI]] = select <vscale x 2 x i1> [[TMP9]], <vscale x 2 x i64> [[TMP10]], <vscale x 2 x i64> [[VEC_PHI]]
+; NO-VP-NEXT: [[PREDPHI2]] = select <vscale x 2 x i1> [[TMP9]], <vscale x 2 x i64> [[TMP12]], <vscale x 2 x i64> [[VEC_PHI1]]
+; NO-VP-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP1]]
+; NO-VP-NEXT: [[VEC_IND_NEXT]] = add <vscale x 2 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
+; NO-VP-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; NO-VP: [[MIDDLE_BLOCK]]:
+; NO-VP-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[PREDPHI]])
+; NO-VP-NEXT: [[TMP15:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64(<vscale x 2 x i64> [[PREDPHI2]])
+; NO-VP-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; NO-VP-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; NO-VP: [[SCALAR_PH]]:
+; NO-VP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP14]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT: [[BC_MERGE_RDX3:%.*]] = phi i64 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; NO-VP-NEXT: br label %[[LOOP:.*]]
+; NO-VP: [[LOOP]]:
+; NO-VP-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; NO-VP-NEXT: [[A:%.*]] = phi i64 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[A_NEXT:%.*]], %[[LATCH]] ]
+; NO-VP-NEXT: [[A2:%.*]] = phi i64 [ [[BC_MERGE_RDX3]], %[[SCALAR_PH]] ], [ [[A2_NEXT:%.*]], %[[LATCH]] ]
+; NO-VP-NEXT: [[GP:%.*]] = getelementptr i32, ptr [[P]], i64 [[IV]]
+; NO-VP-NEXT: [[VP:%.*]] = load i32, ptr [[GP]], align 4
+; NO-VP-NEXT: [[C0:%.*]] = icmp ne i32 [[VP]], 0
+; NO-VP-NEXT: br i1 [[C0]], label %[[IF_OUTER:.*]], label %[[LATCH]]
+; NO-VP: [[IF_OUTER]]:
+; NO-VP-NEXT: [[GQ:%.*]] = getelementptr inbounds i32, ptr [[Q]], i64 [[IV]]
+; NO-VP-NEXT: [[VQ:%.*]] = load i32, ptr [[GQ]], align 4
+; NO-VP-NEXT: [[C1:%.*]] = icmp sgt i32 [[VQ]], 0
+; NO-VP-NEXT: br i1 [[C1]], label %[[IF_INNER:.*]], label %[[LATCH]]
+; NO-VP: [[IF_INNER]]:
+; NO-VP-NEXT: [[ADD:%.*]] = add i64 [[A]], [[IV]]
+; NO-VP-NEXT: [[SQ:%.*]] = mul i64 [[IV]], [[IV]]
+; NO-VP-NEXT: [[ADD2:%.*]] = add i64 [[A2]], [[SQ]]
+; NO-VP-NEXT: br label %[[LATCH]]
+; NO-VP: [[LATCH]]:
+; NO-VP-NEXT: [[A_NEXT]] = phi i64 [ [[A]], %[[LOOP]] ], [ [[A]], %[[IF_OUTER]] ], [ [[ADD]], %[[IF_INNER]] ]
+; NO-VP-NEXT: [[A2_NEXT]] = phi i64 [ [[A2]], %[[LOOP]] ], [ [[A2]], %[[IF_OUTER]] ], [ [[ADD2]], %[[IF_INNER]] ]
+; NO-VP-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
+; NO-VP-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; NO-VP: [[EXIT]]:
+; NO-VP-NEXT: [[A_NEXT_LCSSA:%.*]] = phi i64 [ [[A_NEXT]], %[[LATCH]] ], [ [[TMP14]], %[[MIDDLE_BLOCK]] ]
+; NO-VP-NEXT: [[A2_NEXT_LCSSA:%.*]] = phi i64 [ [[A2_NEXT]], %[[LATCH]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
+; NO-VP-NEXT: store i64 [[A_NEXT_LCSSA]], ptr [[OUT]], align 8
+; NO-VP-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[OUT]], i64 1
+; NO-VP-NEXT: store i64 [[A2_NEXT_LCSSA]], ptr [[O1]], align 8
+; NO-VP-NEXT: ret void
+;
+entry:
+ br label %loop
+
+loop:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ]
+ %a = phi i64 [ 0, %entry ], [ %a.next, %latch ]
+ %a2 = phi i64 [ 0, %entry ], [ %a2.next, %latch ]
+ %gp = getelementptr i32, ptr %p, i64 %iv
+ %vp = load i32, ptr %gp
+ %c0 = icmp ne i32 %vp, 0
+ br i1 %c0, label %if.outer, label %latch
+
+if.outer:
+ %gq = getelementptr inbounds i32, ptr %q, i64 %iv
+ %vq = load i32, ptr %gq
+ %c1 = icmp sgt i32 %vq, 0
+ br i1 %c1, label %if.inner, label %latch
+
+if.inner:
+ %add = add i64 %a, %iv
+ %sq = mul i64 %iv, %iv
+ %add2 = add i64 %a2, %sq
+ br label %latch
+
+latch:
+ %a.next = phi i64 [ %a, %loop ], [ %a, %if.outer ], [ %add, %if.inner ]
+ %a2.next = phi i64 [ %a2, %loop ], [ %a2, %if.outer ], [ %add2, %if.inner ]
+ %iv.next = add i64 %iv, 1
+ %ec = icmp eq i64 %iv.next, %n
+ br i1 %ec, label %exit, label %loop
+
+exit:
+ store i64 %a.next, ptr %out
+ %o1 = getelementptr i64, ptr %out, i64 1
+ store i64 %a2.next, ptr %o1
+ ret void
+}
More information about the llvm-commits
mailing list