[llvm] [VPlan] Move EVL-based transforms to VPlanEVLTransforms.cpp (NFC) (PR #209883)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 17 09:41:38 PDT 2026
https://github.com/fhahn updated https://github.com/llvm/llvm-project/pull/209883
>From 0a3b84038aa589d0ad757f5ffb95df50bd8d2ee9 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Wed, 15 Jul 2026 15:32:26 +0100
Subject: [PATCH 1/4] [VPlan] Move EVL-based transforms to
VPlanEVLTransforms.cpp (NFC)
VPlanTransforms.cpp has become quite big and is one of the top 20 most
expensive files in terms of compile-time.
https://llvm-compile-time-tracker.com/compare_clang.php?from=b8ba3c2b72cb53268129bbecfeb4ba7ec5b8d831&to=96a722a0ace661626d071166635728def97f6820&stat=instructions%3Au
Split off the mostly self-contained EVL related transforms and move them
to a new VPlanEVLTransforms.cpp.
This moves simplifyKnownEVL, optimizeEVLMasks, addExplicitVectorLength,
convertToVariableLengthStep and convertEVLExitCond together with their
EVL-specific static helpers.
Shared helpers used by the moved code (isDeadRecipe,
collectUsersRecursively, recursivelyDeleteDeadRecipes,
getOpcodeOrIntrinsicID and tryToFoldLiveIns, pullOutPermutations) are promoted
to vputils.
---
llvm/lib/Transforms/Vectorize/CMakeLists.txt | 1 +
.../Vectorize/VPlanEVLTransforms.cpp | 657 ++++++++++++++
.../Transforms/Vectorize/VPlanTransforms.cpp | 844 +-----------------
.../Transforms/Vectorize/VPlanTransforms.h | 39 +
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 144 +++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 23 +
6 files changed, 880 insertions(+), 828 deletions(-)
create mode 100644 llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp
diff --git a/llvm/lib/Transforms/Vectorize/CMakeLists.txt b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
index 6e26203d957cb..baacfc9b90a86 100644
--- a/llvm/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/llvm/lib/Transforms/Vectorize/CMakeLists.txt
@@ -30,6 +30,7 @@ add_llvm_component_library(LLVMVectorize
VPlan.cpp
VPlanAnalysis.cpp
VPlanConstruction.cpp
+ VPlanEVLTransforms.cpp
VPlanPredicator.cpp
VPlanRecipes.cpp
VPlanTransforms.cpp
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp
new file mode 100644
index 0000000000000..d1ffa88b7eff6
--- /dev/null
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp
@@ -0,0 +1,657 @@
+//===- VPlanEVLTransforms.cpp - Explicit Vector Length transforms ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the VPlan-to-VPlan transforms related to explicit
+/// vector length (EVL) support.
+///
+//===----------------------------------------------------------------------===//
+
+#include "VPlanTransforms.h"
+#include "LoopVectorizationPlanner.h"
+#include "VPlan.h"
+#include "VPlanCFG.h"
+#include "VPlanHelpers.h"
+#include "VPlanPatternMatch.h"
+#include "VPlanUtils.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/IR/Intrinsics.h"
+
+using namespace llvm;
+using namespace VPlanPatternMatch;
+
+/// From the definition of llvm.experimental.get.vector.length,
+/// VPInstruction::ExplicitVectorLength(%AVL) = %AVL when %AVL <= VF.
+bool VPlanTransforms::simplifyKnownEVL(VPlan &Plan, ElementCount VF,
+ PredicatedScalarEvolution &PSE) {
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getEntry()))) {
+ for (VPRecipeBase &R : *VPBB) {
+ VPValue *AVL;
+ if (!match(&R, m_EVL(m_VPValue(AVL))))
+ continue;
+
+ const SCEV *AVLSCEV = vputils::getSCEVExprForVPValue(AVL, PSE);
+ if (isa<SCEVCouldNotCompute>(AVLSCEV))
+ continue;
+ ScalarEvolution &SE = *PSE.getSE();
+ const SCEV *VFSCEV = SE.getElementCount(AVLSCEV->getType(), VF);
+ if (!SE.isKnownPredicate(CmpInst::ICMP_ULE, AVLSCEV, VFSCEV))
+ continue;
+
+ VPValue *Trunc = VPBuilder(&R).createScalarZExtOrTrunc(
+ AVL, Type::getInt32Ty(Plan.getContext()), AVLSCEV->getType(),
+ R.getDebugLoc());
+ if (Trunc != AVL) {
+ auto *TruncR = cast<VPSingleDefRecipe>(Trunc);
+ const DataLayout &DL = Plan.getDataLayout();
+ if (VPValue *Folded =
+ vputils::tryToFoldLiveIns(*TruncR, TruncR->operands(), DL))
+ Trunc = Folded;
+ }
+ R.getVPSingleValue()->replaceAllUsesWith(Trunc);
+ return true;
+ }
+ }
+ return false;
+}
+
+template <typename Op0_t, typename Op1_t> struct RemoveMask_match {
+ Op0_t In;
+ Op1_t &Out;
+
+ RemoveMask_match(const Op0_t &In, Op1_t &Out) : In(In), Out(Out) {}
+
+ template <typename OpTy> bool match(OpTy *V) const {
+ if (m_Specific(In).match(V)) {
+ Out = nullptr;
+ return true;
+ }
+ return m_LogicalAnd(m_Specific(In), m_VPValue(Out)).match(V);
+ }
+};
+
+/// Match a specific mask \p In, or a combination of it (logical-and In, Out).
+/// Returns the remaining part \p Out if so, or nullptr otherwise.
+template <typename Op0_t, typename Op1_t>
+static inline RemoveMask_match<Op0_t, Op1_t> m_RemoveMask(const Op0_t &In,
+ Op1_t &Out) {
+ return RemoveMask_match<Op0_t, Op1_t>(In, Out);
+}
+
+static std::optional<Intrinsic::ID> getVPDivRemIntrinsic(Intrinsic::ID IntrID) {
+ switch (IntrID) {
+ case Intrinsic::masked_udiv:
+ return Intrinsic::vp_udiv;
+ case Intrinsic::masked_sdiv:
+ return Intrinsic::vp_sdiv;
+ case Intrinsic::masked_urem:
+ return Intrinsic::vp_urem;
+ case Intrinsic::masked_srem:
+ return Intrinsic::vp_srem;
+ default:
+ return std::nullopt;
+ }
+}
+
+/// Try to optimize a \p CurRecipe masked by \p HeaderMask to a corresponding
+/// EVL-based recipe without the header mask. Returns nullptr if no EVL-based
+/// recipe could be created.
+/// \p HeaderMask Header Mask.
+/// \p CurRecipe Recipe to be transform.
+/// \p EVL The explicit vector length parameter of vector-predication
+/// intrinsics.
+static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
+ VPRecipeBase &CurRecipe, VPValue &EVL) {
+ VPlan *Plan = CurRecipe.getParent()->getPlan();
+ DebugLoc DL = CurRecipe.getDebugLoc();
+ VPValue *Addr, *Mask, *EndPtr;
+
+ /// Adjust any end pointers so that they point to the end of EVL lanes not VF.
+ auto AdjustEndPtr = [&CurRecipe, &EVL](VPValue *EndPtr) {
+ auto *EVLEndPtr = cast<VPVectorEndPointerRecipe>(EndPtr)->clone();
+ EVLEndPtr->insertBefore(&CurRecipe);
+ // Cast EVL (i32) to match the VF operand's type.
+ VPValue *EVLAsVF = VPBuilder(EVLEndPtr).createScalarZExtOrTrunc(
+ &EVL, EVLEndPtr->getOperand(1)->getScalarType(), EVL.getScalarType(),
+ DebugLoc::getUnknown());
+ EVLEndPtr->setOperand(1, EVLAsVF);
+ return EVLEndPtr;
+ };
+
+ auto GetVPReverse = [&CurRecipe, &EVL, Plan,
+ DL](VPValue *V) -> VPWidenIntrinsicRecipe * {
+ if (!V)
+ return nullptr;
+ auto *Reverse = new VPWidenIntrinsicRecipe(
+ Intrinsic::experimental_vp_reverse, {V, Plan->getTrue(), &EVL},
+ V->getScalarType(), {}, {}, DL);
+ Reverse->insertBefore(&CurRecipe);
+ return Reverse;
+ };
+
+ if (match(&CurRecipe,
+ m_MaskedLoad(m_VPValue(Addr), m_RemoveMask(HeaderMask, Mask))))
+ return new VPWidenLoadEVLRecipe(cast<VPWidenLoadRecipe>(CurRecipe), Addr,
+ EVL, Mask);
+
+ if (match(&CurRecipe,
+ m_MaskedLoad(m_VPValue(EndPtr),
+ m_Reverse(m_RemoveMask(HeaderMask, Mask)))) &&
+ match(EndPtr, m_VecEndPtr(m_VPValue(), m_Specific(&Plan->getVF())))) {
+ Mask = GetVPReverse(Mask);
+ Addr = AdjustEndPtr(EndPtr);
+ auto *LoadR = new VPWidenLoadEVLRecipe(cast<VPWidenLoadRecipe>(CurRecipe),
+ Addr, EVL, Mask);
+ LoadR->insertBefore(&CurRecipe);
+ VPValue *Poison = Plan->getPoison(LoadR->getScalarType());
+ return new VPWidenIntrinsicRecipe(Intrinsic::vector_splice_left,
+ {Poison, LoadR, &EVL},
+ LoadR->getScalarType(), {}, {}, DL);
+ }
+
+ VPValue *Stride;
+ if (match(&CurRecipe, m_Intrinsic<Intrinsic::experimental_vp_strided_load>(
+ m_VPValue(Addr), m_VPValue(Stride),
+ m_RemoveMask(HeaderMask, Mask),
+ m_TruncOrSelf(m_Specific(&Plan->getVF()))))) {
+ if (!Mask)
+ Mask = Plan->getTrue();
+ auto *NewLoad = cast<VPWidenMemIntrinsicRecipe>(&CurRecipe)->clone();
+ NewLoad->setOperand(2, Mask);
+ NewLoad->setOperand(3, &EVL);
+ return NewLoad;
+ }
+
+ VPValue *StoredVal;
+ if (match(&CurRecipe, m_MaskedStore(m_VPValue(Addr), m_VPValue(StoredVal),
+ m_RemoveMask(HeaderMask, Mask))))
+ return new VPWidenStoreEVLRecipe(cast<VPWidenStoreRecipe>(CurRecipe), Addr,
+ StoredVal, EVL, Mask);
+
+ if (match(&CurRecipe,
+ m_MaskedStore(m_VPValue(EndPtr), m_VPValue(StoredVal),
+ m_Reverse(m_RemoveMask(HeaderMask, Mask)))) &&
+ match(EndPtr, m_VecEndPtr(m_VPValue(), m_Specific(&Plan->getVF())))) {
+ Mask = GetVPReverse(Mask);
+ Addr = AdjustEndPtr(EndPtr);
+ VPValue *Poison = Plan->getPoison(StoredVal->getScalarType());
+ auto *SpliceR = new VPWidenIntrinsicRecipe(
+ Intrinsic::vector_splice_right, {StoredVal, Poison, &EVL},
+ StoredVal->getScalarType(), {}, {}, DL);
+ SpliceR->insertBefore(&CurRecipe);
+ return new VPWidenStoreEVLRecipe(cast<VPWidenStoreRecipe>(CurRecipe), Addr,
+ SpliceR, EVL, Mask);
+ }
+
+ if (auto *Rdx = dyn_cast<VPReductionRecipe>(&CurRecipe))
+ if (Rdx->isConditional() &&
+ match(Rdx->getCondOp(), m_RemoveMask(HeaderMask, Mask)))
+ return new VPReductionEVLRecipe(*Rdx, EVL, Mask);
+
+ if (auto *Interleave = dyn_cast<VPInterleaveRecipe>(&CurRecipe))
+ if (Interleave->getMask() &&
+ match(Interleave->getMask(), m_RemoveMask(HeaderMask, Mask)))
+ return new VPInterleaveEVLRecipe(*Interleave, EVL, Mask);
+
+ VPValue *LHS, *RHS;
+ if (match(&CurRecipe, m_SelectLike(m_RemoveMask(HeaderMask, Mask),
+ m_VPValue(LHS), m_VPValue(RHS))))
+ return new VPWidenIntrinsicRecipe(
+ Intrinsic::vp_merge, {Mask ? Mask : Plan->getTrue(), LHS, RHS, &EVL},
+ LHS->getScalarType(), {}, {}, DL);
+
+ if (match(&CurRecipe, m_LastActiveLane(m_Specific(HeaderMask)))) {
+ Type *Ty = CurRecipe.getVPSingleValue()->getScalarType();
+ VPValue *ZExt =
+ VPBuilder(&CurRecipe)
+ .createScalarZExtOrTrunc(&EVL, Ty, EVL.getScalarType(), DL);
+ return new VPInstruction(
+ Instruction::Sub, {ZExt, Plan->getConstantInt(Ty, 1)},
+ VPIRFlags::getDefaultFlags(Instruction::Sub), {}, DL);
+ }
+
+ // lhs | (headermask && rhs) -> vp.merge rhs, true, lhs, evl
+ if (match(&CurRecipe,
+ m_c_BinaryOr(m_VPValue(LHS),
+ m_LogicalAnd(m_Specific(HeaderMask), m_VPValue(RHS)))))
+ return new VPWidenIntrinsicRecipe(Intrinsic::vp_merge,
+ {RHS, Plan->getTrue(), LHS, &EVL},
+ LHS->getScalarType(), {}, {}, DL);
+
+ if (auto *IntrR = dyn_cast<VPWidenIntrinsicRecipe>(&CurRecipe))
+ if (auto VPID = getVPDivRemIntrinsic(IntrR->getVectorIntrinsicID()))
+ if (match(IntrR->getOperand(2), m_RemoveMask(HeaderMask, Mask)))
+ return new VPWidenIntrinsicRecipe(*VPID,
+ {IntrR->getOperand(0),
+ IntrR->getOperand(1),
+ Mask ? Mask : Plan->getTrue(), &EVL},
+ IntrR->getScalarType(), {}, {}, DL);
+
+ return nullptr;
+}
+
+/// Optimize away any EVL-based header masks to VP intrinsic based recipes.
+/// The transforms here need to preserve the original semantics.
+void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) {
+ // Find the EVL-based header mask if it exists: icmp ult step-vector, EVL
+ VPValue *HeaderMask = nullptr, *EVL = nullptr;
+ for (VPRecipeBase &R : *Plan.getVectorLoopRegion()->getEntryBasicBlock()) {
+ if (match(&R, m_SpecificICmp(CmpInst::ICMP_ULT, m_StepVector(),
+ m_VPValue(EVL))) &&
+ match(EVL, m_EVL(m_VPValue()))) {
+ HeaderMask = R.getVPSingleValue();
+ break;
+ }
+ }
+ if (!HeaderMask)
+ return;
+
+ SmallVector<VPRecipeBase *> OldRecipes;
+ for (VPUser *U : vputils::collectUsersRecursively(HeaderMask)) {
+ VPRecipeBase *R = cast<VPRecipeBase>(U);
+ if (auto *NewR = optimizeMaskToEVL(HeaderMask, *R, *EVL)) {
+ NewR->insertBefore(R);
+ for (auto [Old, New] :
+ zip_equal(R->definedValues(), NewR->definedValues()))
+ Old->replaceAllUsesWith(New);
+ OldRecipes.push_back(R);
+ }
+ }
+
+ // Replace remaining (HeaderMask && Mask) with vp.merge (True, Mask,
+ // False, EVL)
+ for (VPUser *U : vputils::collectUsersRecursively(HeaderMask)) {
+ VPValue *Mask;
+ if (match(U, m_LogicalAnd(m_Specific(HeaderMask), m_VPValue(Mask)))) {
+ auto *LogicalAnd = cast<VPInstruction>(U);
+ auto *Merge = new VPWidenIntrinsicRecipe(
+ Intrinsic::vp_merge, {Plan.getTrue(), Mask, Plan.getFalse(), EVL},
+ Mask->getScalarType(), {}, {}, LogicalAnd->getDebugLoc());
+ Merge->insertBefore(LogicalAnd);
+ LogicalAnd->replaceAllUsesWith(Merge);
+ OldRecipes.push_back(LogicalAnd);
+ }
+ }
+
+ // Pull out left splices from any elementwise op.
+ // binop(splice.left(poison, x, evl), live-in)
+ // -> splice.left(poison, binop(x,live-in), evl)
+ pullOutPermutations(
+ Plan,
+ [&EVL](const auto &X) {
+ return m_Intrinsic<Intrinsic::vector_splice_left>(m_Poison(), X,
+ m_Specific(EVL));
+ },
+ [&Plan, &EVL](auto *X) {
+ return new VPWidenIntrinsicRecipe(
+ Intrinsic::vector_splice_left,
+ {Plan.getPoison(X->getScalarType()), X, EVL}, X->getScalarType(),
+ {}, {}, X->getDebugLoc());
+ });
+
+ // Fold the following splice patterns:
+ // splice.right(splice.left(poison, x, evl), poison, evl) -> x
+ // vector.reverse(splice.left(poison, x, evl)) -> vp.reverse(x, true, evl)
+ // splice.right(vector.reverse(x), poison, evl) -> vp.reverse(x, true, evl)
+ for (VPUser *U : vputils::collectUsersRecursively(EVL)) {
+ auto *R = cast<VPRecipeBase>(U);
+ // Remove potentially dead left splices from the transform above.
+ if (match(U, m_Intrinsic<Intrinsic::vector_splice_left>()) &&
+ R->getVPSingleValue()->getNumUsers() == 0) {
+ OldRecipes.push_back(R);
+ continue;
+ }
+
+ VPValue *X;
+ if (match(U, m_Intrinsic<Intrinsic::vector_splice_right>(
+ m_Intrinsic<Intrinsic::vector_splice_left>(
+ m_Poison(), m_VPValue(X), m_Specific(EVL)),
+ m_Poison(), m_Specific(EVL)))) {
+ R->getVPSingleValue()->replaceAllUsesWith(X);
+ OldRecipes.push_back(R);
+ continue;
+ }
+
+ if (!match(U,
+ m_CombineOr(
+ m_Reverse(m_Intrinsic<Intrinsic::vector_splice_left>(
+ m_Poison(), m_VPValue(X), m_Specific(EVL))),
+ m_Intrinsic<Intrinsic::vector_splice_right>(
+ m_Reverse(m_VPValue(X)), m_Poison(), m_Specific(EVL)))))
+ continue;
+
+ auto *VPReverse = new VPWidenIntrinsicRecipe(
+ Intrinsic::experimental_vp_reverse, {X, Plan.getTrue(), EVL},
+ X->getScalarType(), {}, {}, R->getDebugLoc());
+ VPReverse->insertBefore(R);
+ R->getVPSingleValue()->replaceAllUsesWith(VPReverse);
+ OldRecipes.push_back(R);
+ }
+
+ for (VPRecipeBase *R : reverse(OldRecipes)) {
+ SmallVector<VPValue *> PossiblyDead(R->operands());
+ R->eraseFromParent();
+ for (VPValue *Op : PossiblyDead)
+ vputils::recursivelyDeleteDeadRecipes(Op);
+ }
+}
+
+/// After replacing the canonical IV with a EVL-based IV, fixup recipes that use
+/// VF to use the EVL instead to avoid incorrect updates on the penultimate
+/// iteration.
+static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
+
+ // EVL is i32 but VF/VFxUF are IdxTy. Convert as needed.
+ VPValue *EVLAsIdx =
+ VPBuilder::getToInsertAfter(EVL.getDefiningRecipe())
+ .createScalarZExtOrTrunc(&EVL, Plan.getVF().getScalarType(),
+ EVL.getScalarType(), DebugLoc::getUnknown());
+
+ assert(all_of(Plan.getVF().users(),
+ [&Plan](VPUser *U) {
+ auto IsAllowedUser =
+ IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
+ VPWidenIntOrFpInductionRecipe,
+ VPWidenMemIntrinsicRecipe>;
+ if (match(U, m_Trunc(m_Specific(&Plan.getVF()))))
+ return all_of(cast<VPSingleDefRecipe>(U)->users(),
+ IsAllowedUser);
+ return IsAllowedUser(U);
+ }) &&
+ "User of VF that we can't transform to EVL.");
+ Plan.getVF().replaceUsesWithIf(EVLAsIdx, [](VPUser &U, unsigned Idx) {
+ return isa<VPWidenIntOrFpInductionRecipe, VPScalarIVStepsRecipe>(U);
+ });
+
+ assert(all_of(Plan.getVFxUF().users(),
+ match_fn(m_CombineOr(
+ m_c_Add(m_Specific(LoopRegion->getCanonicalIV()),
+ m_Specific(&Plan.getVFxUF())),
+ m_Isa<VPWidenPointerInductionRecipe>()))) &&
+ "Only users of VFxUF should be VPWidenPointerInductionRecipe and the "
+ "increment of the canonical induction.");
+ Plan.getVFxUF().replaceUsesWithIf(EVLAsIdx, [](VPUser &U, unsigned Idx) {
+ // Only replace uses in VPWidenPointerInductionRecipe; The increment of the
+ // canonical induction must not be updated.
+ return isa<VPWidenPointerInductionRecipe>(U);
+ });
+
+ // Create a scalar phi to track the previous EVL if fixed-order recurrence is
+ // contained.
+ bool ContainsFORs =
+ any_of(Header->phis(), IsaPred<VPFirstOrderRecurrencePHIRecipe>);
+ if (ContainsFORs) {
+ // TODO: Use VPInstruction::ExplicitVectorLength to get maximum EVL.
+ VPValue *MaxEVL = &Plan.getVF();
+ // Emit VPScalarCastRecipe in preheader if VF is not a 32 bits integer.
+ VPBuilder Builder(LoopRegion->getPreheaderVPBB());
+ MaxEVL = Builder.createScalarZExtOrTrunc(
+ MaxEVL, Type::getInt32Ty(Plan.getContext()), MaxEVL->getScalarType(),
+ DebugLoc::getUnknown());
+
+ Builder.setInsertPoint(Header, Header->getFirstNonPhi());
+ VPValue *PrevEVL = Builder.createScalarPhi(
+ {MaxEVL, &EVL}, DebugLoc::getUnknown(), "prev.evl");
+
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
+ for (VPRecipeBase &R : *VPBB) {
+ VPValue *V1, *V2;
+ if (!match(&R,
+ m_VPInstruction<VPInstruction::FirstOrderRecurrenceSplice>(
+ m_VPValue(V1), m_VPValue(V2))))
+ continue;
+ VPValue *Imm = Plan.getOrAddLiveIn(
+ ConstantInt::getSigned(Type::getInt32Ty(Plan.getContext()), -1));
+ VPWidenIntrinsicRecipe *VPSplice = new VPWidenIntrinsicRecipe(
+ Intrinsic::experimental_vp_splice,
+ {V1, V2, Imm, Plan.getTrue(), PrevEVL, &EVL},
+ R.getVPSingleValue()->getScalarType(), {}, {}, R.getDebugLoc());
+ VPSplice->insertBefore(&R);
+ R.getVPSingleValue()->replaceAllUsesWith(VPSplice);
+ }
+ }
+ }
+
+ VPValue *HeaderMask = LoopRegion->getHeaderMask();
+ if (!HeaderMask)
+ return;
+
+ // Ensure that any reduction that uses a select to mask off tail lanes does so
+ // in the vector loop, not the middle block, since EVL tail folding can have
+ // tail elements in the penultimate iteration.
+ assert(all_of(*Plan.getMiddleBlock(), [&Plan, HeaderMask](VPRecipeBase &R) {
+ if (match(&R, m_ComputeReductionResult(m_Select(m_Specific(HeaderMask),
+ m_VPValue(), m_VPValue()))))
+ return R.getOperand(0)->getDefiningRecipe()->getRegion() ==
+ Plan.getVectorLoopRegion();
+ return true;
+ }));
+
+ // Replace the abstract header mask with a mask equivalent to predicating by
+ // EVL: icmp ult step-vector, EVL
+ VPRecipeBase *EVLR = EVL.getDefiningRecipe();
+ VPBuilder Builder(EVLR->getParent(), std::next(EVLR->getIterator()));
+ Type *EVLType = EVL.getScalarType();
+ VPValue *EVLMask = Builder.createICmp(
+ CmpInst::ICMP_ULT,
+ Builder.createNaryOp(VPInstruction::StepVector, {}, EVLType), &EVL);
+ HeaderMask->replaceAllUsesWith(EVLMask);
+}
+
+/// Converts a tail folded vector loop region to step by
+/// VPInstruction::ExplicitVectorLength elements instead of VF elements each
+/// iteration.
+///
+/// - Add a VPCurrentIterationPHIRecipe and related recipes to \p Plan and
+/// replaces all uses of the canonical IV except for the canonical IV
+/// increment with a VPCurrentIterationPHIRecipe. The canonical IV is used
+/// only for loop iterations counting after this transformation.
+///
+/// - The header mask is replaced with a header mask based on the EVL.
+///
+/// - Plans with FORs have a new phi added to keep track of the EVL of the
+/// previous iteration, and VPFirstOrderRecurrencePHIRecipes are replaced with
+/// @llvm.vp.splice.
+///
+/// The function uses the following definitions:
+/// %StartV is the canonical induction start value.
+///
+/// The function adds the following recipes:
+///
+/// vector.ph:
+/// ...
+///
+/// vector.body:
+/// ...
+/// %CurrentIter = CURRENT-ITERATION-PHI [ %StartV, %vector.ph ],
+/// [ %NextIter, %vector.body ]
+/// %AVL = phi [ trip-count, %vector.ph ], [ %NextAVL, %vector.body ]
+/// %VPEVL = EXPLICIT-VECTOR-LENGTH %AVL
+/// ...
+/// %OpEVL = cast i32 %VPEVL to IVSize
+/// %NextIter = add IVSize %OpEVL, %CurrentIter
+/// %NextAVL = sub IVSize nuw %AVL, %OpEVL
+/// ...
+///
+/// If MaxSafeElements is provided, the function adds the following recipes:
+/// vector.ph:
+/// ...
+///
+/// vector.body:
+/// ...
+/// %CurrentIter = CURRENT-ITERATION-PHI [ %StartV, %vector.ph ],
+/// [ %NextIter, %vector.body ]
+/// %AVL = phi [ trip-count, %vector.ph ], [ %NextAVL, %vector.body ]
+/// %cmp = cmp ult %AVL, MaxSafeElements
+/// %SAFE_AVL = select %cmp, %AVL, MaxSafeElements
+/// %VPEVL = EXPLICIT-VECTOR-LENGTH %SAFE_AVL
+/// ...
+/// %OpEVL = cast i32 %VPEVL to IVSize
+/// %NextIter = add IVSize %OpEVL, %CurrentIter
+/// %NextAVL = sub IVSize nuw %AVL, %OpEVL
+/// ...
+///
+void VPlanTransforms::addExplicitVectorLength(
+ VPlan &Plan, const std::optional<unsigned> &MaxSafeElements) {
+ if (Plan.hasScalarVFOnly())
+ return;
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
+
+ auto *CanonicalIV = LoopRegion->getCanonicalIV();
+ auto *CanIVTy = LoopRegion->getCanonicalIVType();
+ VPValue *StartV = Plan.getZero(CanIVTy);
+ auto *CanonicalIVIncrement = LoopRegion->getOrCreateCanonicalIVIncrement();
+
+ // Create the CurrentIteration recipe in the vector loop.
+ auto *CurrentIteration =
+ new VPCurrentIterationPHIRecipe(StartV, DebugLoc::getUnknown());
+ CurrentIteration->insertBefore(*Header, Header->begin());
+ VPBuilder Builder(Header, Header->getFirstNonPhi());
+ // Create the AVL (application vector length), starting from TC -> 0 in steps
+ // of EVL.
+ VPPhi *AVLPhi = Builder.createScalarPhi(
+ {Plan.getTripCount()}, DebugLoc::getCompilerGenerated(), "avl");
+ VPValue *AVL = AVLPhi;
+
+ if (MaxSafeElements) {
+ // Support for MaxSafeDist for correct loop emission.
+ VPValue *AVLSafe = Plan.getConstantInt(CanIVTy, *MaxSafeElements);
+ VPValue *Cmp = Builder.createICmp(ICmpInst::ICMP_ULT, AVL, AVLSafe);
+ AVL = Builder.createSelect(Cmp, AVL, AVLSafe, DebugLoc::getUnknown(),
+ "safe_avl");
+ }
+ auto *VPEVL = Builder.createNaryOp(VPInstruction::ExplicitVectorLength, AVL,
+ DebugLoc::getUnknown(), "evl");
+
+ Builder.setInsertPoint(CanonicalIVIncrement);
+ VPValue *OpVPEVL = VPEVL;
+
+ auto *I32Ty = Type::getInt32Ty(Plan.getContext());
+ OpVPEVL = Builder.createScalarZExtOrTrunc(
+ OpVPEVL, CanIVTy, I32Ty, CanonicalIVIncrement->getDebugLoc());
+
+ auto *NextIter = Builder.createAdd(
+ OpVPEVL, CurrentIteration, CanonicalIVIncrement->getDebugLoc(),
+ "current.iteration.next", CanonicalIVIncrement->getNoWrapFlags());
+ CurrentIteration->addBackedgeValue(NextIter);
+
+ VPValue *NextAVL =
+ Builder.createSub(AVLPhi, OpVPEVL, DebugLoc::getCompilerGenerated(),
+ "avl.next", {/*NUW=*/true, /*NSW=*/false});
+ AVLPhi->addIncoming(NextAVL);
+
+ fixupVFUsersForEVL(Plan, *VPEVL);
+ removeDeadRecipes(Plan);
+
+ // Replace all uses of the canonical IV with VPCurrentIterationPHIRecipe
+ // except for the canonical IV increment.
+ CanonicalIV->replaceUsesWithIf(CurrentIteration,
+ [CanonicalIVIncrement](VPUser &U, unsigned) {
+ return &U != CanonicalIVIncrement;
+ });
+ // TODO: support unroll factor > 1.
+ Plan.setUF(1);
+}
+
+void VPlanTransforms::convertToVariableLengthStep(VPlan &Plan) {
+ // Find the vector loop entry by locating VPCurrentIterationPHIRecipe.
+ // There should be only one VPCurrentIteration in the entire plan.
+ VPCurrentIterationPHIRecipe *CurrentIteration = nullptr;
+
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
+ vp_depth_first_shallow(Plan.getEntry())))
+ for (VPRecipeBase &R : VPBB->phis())
+ if (auto *PhiR = dyn_cast<VPCurrentIterationPHIRecipe>(&R)) {
+ assert(!CurrentIteration &&
+ "Found multiple CurrentIteration. Only one expected");
+ CurrentIteration = PhiR;
+ }
+
+ // Early return if it is not variable-length stepping.
+ if (!CurrentIteration)
+ return;
+
+ VPBasicBlock *HeaderVPBB = CurrentIteration->getParent();
+ VPValue *CurrentIterationIncr = CurrentIteration->getBackedgeValue();
+
+ // Convert CurrentIteration to concrete recipe.
+ auto *ScalarR =
+ VPBuilder(CurrentIteration)
+ .createScalarPhi(
+ {CurrentIteration->getStartValue(), CurrentIterationIncr},
+ CurrentIteration->getDebugLoc(), "current.iteration.iv");
+ CurrentIteration->replaceAllUsesWith(ScalarR);
+ CurrentIteration->eraseFromParent();
+
+ // Replace CanonicalIVInc with CurrentIteration increment if it exists.
+ auto *CanonicalIV = cast<VPPhi>(&*HeaderVPBB->begin());
+ if (auto *CanIVInc = findUserOf(
+ CanonicalIV, m_c_Add(m_VPValue(), m_Specific(&Plan.getVFxUF())))) {
+ cast<VPInstruction>(CanIVInc)->replaceAllUsesWith(CurrentIterationIncr);
+ CanIVInc->eraseFromParent();
+ }
+}
+
+void VPlanTransforms::convertEVLExitCond(VPlan &Plan) {
+ VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
+ if (!LoopRegion)
+ return;
+ VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
+ if (Header->empty())
+ return;
+ // The EVL IV is always at the beginning.
+ auto *EVLPhi = dyn_cast<VPCurrentIterationPHIRecipe>(&Header->front());
+ if (!EVLPhi)
+ return;
+
+ // Bail if not an EVL tail folded loop.
+ VPValue *AVL;
+ if (!match(EVLPhi->getBackedgeValue(),
+ m_c_Add(m_ZExtOrSelf(m_EVL(m_VPValue(AVL))), m_Specific(EVLPhi))))
+ return;
+
+ // The AVL may be capped to a safe distance.
+ VPValue *SafeAVL, *UnsafeAVL;
+ if (match(AVL,
+ m_Select(m_SpecificICmp(CmpInst::ICMP_ULT, m_VPValue(UnsafeAVL),
+ m_VPValue(SafeAVL)),
+ m_Deferred(UnsafeAVL), m_Deferred(SafeAVL))))
+ AVL = UnsafeAVL;
+
+ VPValue *AVLNext;
+ [[maybe_unused]] bool FoundAVLNext =
+ match(AVL, m_VPInstruction<Instruction::PHI>(
+ m_Specific(Plan.getTripCount()), m_VPValue(AVLNext)));
+ assert(FoundAVLNext && "Didn't find AVL backedge?");
+
+ VPBasicBlock *Latch = LoopRegion->getExitingBasicBlock();
+ auto *LatchBr = cast<VPInstruction>(Latch->getTerminator());
+ if (match(LatchBr, m_BranchOnCond(m_True())))
+ return;
+
+ VPValue *CanIVInc;
+ [[maybe_unused]] bool FoundIncrement = match(
+ LatchBr,
+ m_BranchOnCond(m_SpecificCmp(CmpInst::ICMP_EQ, m_VPValue(CanIVInc),
+ m_Specific(&Plan.getVectorTripCount()))));
+ assert(FoundIncrement &&
+ match(CanIVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()),
+ m_Specific(&Plan.getVFxUF()))) &&
+ "Expected BranchOnCond with ICmp comparing CanIV + VFxUF with vector "
+ "trip count");
+
+ Type *AVLTy = AVLNext->getScalarType();
+ VPBuilder Builder(LatchBr);
+ LatchBr->setOperand(
+ 0, Builder.createICmp(CmpInst::ICMP_EQ, AVLNext, Plan.getZero(AVLTy)));
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 47e51f9639903..c25ad716db5d4 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -28,7 +28,6 @@
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Analysis/IVDescriptors.h"
-#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
@@ -791,23 +790,6 @@ void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
WideCanIV->eraseFromParent();
}
-/// Returns true if \p R is dead and can be removed.
-static bool isDeadRecipe(VPRecipeBase &R) {
- // Do remove conditional assume instructions as their conditions may be
- // flattened.
- auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
- bool IsConditionalAssume = RepR && RepR->isPredicated() &&
- match(RepR, m_Intrinsic<Intrinsic::assume>());
- if (IsConditionalAssume)
- return true;
-
- if (R.mayHaveSideEffects())
- return false;
-
- // Recipe is dead if no user keeps the recipe alive.
- return all_of(R.definedValues(), [](VPValue *V) { return V->user_empty(); });
-}
-
void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
Plan.getEntry());
@@ -815,7 +797,7 @@ void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
// The recipes in the block are processed in reverse order, to catch chains
// of dead recipes.
for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
- if (isDeadRecipe(R)) {
+ if (vputils::isDeadRecipe(R)) {
R.eraseFromParent();
continue;
}
@@ -838,16 +820,6 @@ void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
}
}
-static SmallVector<VPUser *> collectUsersRecursively(VPValue *V) {
- SetVector<VPUser *> Users(llvm::from_range, V->users());
- for (unsigned I = 0; I != Users.size(); ++I) {
- VPRecipeBase *Cur = cast<VPRecipeBase>(Users[I]);
- for (VPValue *V : Cur->definedValues())
- Users.insert_range(V->users());
- }
- return Users.takeVector();
-}
-
/// Scalarize a VPWidenPointerInductionRecipe by replacing it with a PtrAdd
/// (IndStart, ScalarIVSteps (0, Step)). This is used when the recipe only
/// generates scalar values.
@@ -890,7 +862,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
// VPlan analysis.
// TODO: Apply to all recipes in the future, to replace legacy uniformity
// analysis.
- auto Users = collectUsersRecursively(PhiR);
+ auto Users = vputils::collectUsersRecursively(PhiR);
for (VPUser *U : reverse(Users)) {
auto *Def = dyn_cast<VPRecipeWithIRFlags>(U);
auto *RepR = dyn_cast<VPReplicateRecipe>(U);
@@ -1230,128 +1202,6 @@ static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
}
}
-static void recursivelyDeleteDeadRecipes(VPValue *V) {
- SmallVector<VPValue *> WorkList;
- SmallPtrSet<VPValue *, 8> Seen;
- WorkList.push_back(V);
-
- while (!WorkList.empty()) {
- VPValue *Cur = WorkList.pop_back_val();
- if (!Seen.insert(Cur).second)
- continue;
- VPRecipeBase *R = Cur->getDefiningRecipe();
- if (!R)
- continue;
- if (!isDeadRecipe(*R))
- continue;
- append_range(WorkList, R->operands());
- R->eraseFromParent();
- }
-}
-
-/// Get any instruction opcode or intrinsic ID data embedded in recipe \p R.
-/// Returns an optional pair, where the first element indicates whether it is
-/// an intrinsic ID.
-static std::optional<std::pair<bool, unsigned>>
-getOpcodeOrIntrinsicID(const VPSingleDefRecipe *R) {
- if (Intrinsic::ID IID = vputils::getIntrinsicID(R))
- return std::make_pair(true, IID);
- return TypeSwitch<const VPSingleDefRecipe *,
- std::optional<std::pair<bool, unsigned>>>(R)
- .Case<VPInstruction, VPWidenRecipe, VPWidenCastRecipe, VPWidenGEPRecipe,
- VPReplicateRecipe>(
- [](auto *I) { return std::make_pair(false, I->getOpcode()); })
- .Case([](const VPWidenPHIRecipe *I) {
- return std::make_pair(false, Instruction::PHI);
- })
- .Case<VPVectorPointerRecipe, VPPredInstPHIRecipe, VPScalarIVStepsRecipe>(
- [](auto *I) {
- // For recipes that do not directly map to LLVM IR instructions,
- // assign opcodes after the last VPInstruction opcode (which is also
- // after the last IR Instruction opcode), based on the VPRecipeID.
- return std::make_pair(false, VPInstruction::OpsEnd + 1 +
- I->getVPRecipeID());
- })
- .Default([](auto *) { return std::nullopt; });
-}
-
-/// Try to fold \p R using InstSimplifyFolder. Will succeed and return a
-/// non-nullptr VPValue for a handled opcode or intrinsic ID if corresponding \p
-/// Operands are foldable live-ins.
-static VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R,
- ArrayRef<VPValue *> Operands,
- const DataLayout &DL) {
- auto OpcodeOrIID = getOpcodeOrIntrinsicID(&R);
- if (!OpcodeOrIID)
- return nullptr;
-
- SmallVector<Value *, 4> Ops;
- for (VPValue *Op : Operands) {
- VPValue *Candidate = Op;
- match(Op, m_Broadcast(m_VPValue(Candidate)));
- if (!match(Candidate, m_LiveIn()))
- return nullptr;
- Value *V = Candidate->getUnderlyingValue();
- if (!V)
- return nullptr;
- Ops.push_back(V);
- }
-
- VPlan &Plan = *R.getParent()->getPlan();
- auto FoldToIRValue = [&]() -> Value * {
- InstSimplifyFolder Folder(DL);
- if (OpcodeOrIID->first) {
- // VPInstructions store the called intrinsic as last operand.
- if (isa<VPInstruction>(R))
- Ops.pop_back();
-
- auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(&R);
- return Folder.FoldIntrinsic(OpcodeOrIID->second, Ops, R.getScalarType(),
- RFlags ? RFlags->getFastMathFlagsOrNone()
- : FastMathFlags());
- }
- unsigned Opcode = OpcodeOrIID->second;
- if (Instruction::isBinaryOp(Opcode))
- return Folder.FoldBinOp(static_cast<Instruction::BinaryOps>(Opcode),
- Ops[0], Ops[1]);
- if (Instruction::isCast(Opcode))
- return Folder.FoldCast(static_cast<Instruction::CastOps>(Opcode), Ops[0],
- R.getVPSingleValue()->getScalarType());
- switch (Opcode) {
- case VPInstruction::Not:
- return Folder.FoldBinOp(Instruction::BinaryOps::Xor, Ops[0],
- Constant::getAllOnesValue(Ops[0]->getType()));
- case Instruction::Select:
- return Folder.FoldSelect(Ops[0], Ops[1], Ops[2]);
- case Instruction::ICmp:
- case Instruction::FCmp:
- return Folder.FoldCmp(cast<VPRecipeWithIRFlags>(R).getPredicate(), Ops[0],
- Ops[1]);
- case Instruction::GetElementPtr: {
- auto &RFlags = cast<VPRecipeWithIRFlags>(R);
- auto *GEP = cast<GetElementPtrInst>(RFlags.getUnderlyingInstr());
- return Folder.FoldGEP(GEP->getSourceElementType(), Ops[0],
- drop_begin(Ops), RFlags.getGEPNoWrapFlags());
- }
- case VPInstruction::PtrAdd:
- case VPInstruction::WidePtrAdd:
- return Folder.FoldGEP(IntegerType::getInt8Ty(Plan.getContext()), Ops[0],
- Ops[1],
- cast<VPRecipeWithIRFlags>(R).getGEPNoWrapFlags());
- // An extract of a live-in is an extract of a broadcast, so return the
- // broadcasted element.
- case Instruction::ExtractElement:
- assert(!Ops[0]->getType()->isVectorTy() && "Live-ins should be scalar");
- return Ops[0];
- }
- return nullptr;
- };
-
- if (Value *V = FoldToIRValue())
- return Plan.getOrAddLiveIn(V);
- return nullptr;
-}
-
/// Try to simplify logical and bitwise recipes in \p Def.
static bool simplifyLogicalRecipe(VPSingleDefRecipe *Def, VPBuilder &Builder,
bool CanCreateNewRecipe) {
@@ -1487,7 +1337,7 @@ static void simplifyRecipe(VPSingleDefRecipe *Def) {
// Simplification of live-in IR values for SingleDef recipes using
// InstSimplifyFolder.
const DataLayout &DL = Plan->getDataLayout();
- if (VPValue *V = tryToFoldLiveIns(*Def, Def->operands(), DL))
+ if (VPValue *V = vputils::tryToFoldLiveIns(*Def, Def->operands(), DL))
return Def->replaceAllUsesWith(V);
// Fold PredPHI LiveIn -> LiveIn.
@@ -1887,42 +1737,6 @@ void VPlanTransforms::simplifyRecipes(VPlan &Plan) {
}
}
-/// Removes the permutation pattern \p Perm from any elementwise operations
-/// in the plan, by constructing a new permutation via \p Build.
-/// e.g. binop(perm(x), perm(y)) -> perm(binop(x,y)).
-template <typename Match_t, typename Builder>
-static void pullOutPermutations(VPlan &Plan, Match_t Perm, Builder Build) {
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_deep(Plan.getEntry()))) {
- for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
- if (!Def || !vputils::isElementwise(Def))
- continue;
-
- // At least one of the ops must be a permutation.
- if (!any_of(Def->operands(), match_fn(Perm(m_VPValue()))))
- continue;
-
- // All operands must be permuted or a live in (splat).
- if (!all_of(
- Def->operands(),
- match_fn(m_CombineOr(m_OneUse(Perm(m_VPValue())), m_LiveIn()))))
- continue;
-
- VPValue *X;
- // Remove the inner permutations.
- for (unsigned I = 0; I < Def->getNumOperands(); I++)
- if (match(Def->getOperand(I), Perm(m_VPValue(X))))
- Def->setOperand(I, X);
-
- VPSingleDefRecipe *Res = Build(Def);
- Res->insertAfter(Def);
- Def->replaceUsesWithIf(
- Res, [&Res](VPUser &U, unsigned _) { return &U != Res; });
- }
- }
-}
-
void VPlanTransforms::simplifyReverses(VPlan &Plan) {
// Pull out reverses from any elementwise op.
// binop(reverse(x), reverse(y)) -> reverse(binop(x,y))
@@ -2069,12 +1883,13 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) {
continue;
auto *Clone = VPBuilder::createSingleScalarOp(
- getOpcodeOrIntrinsicID(RepOrWidenR)->second, RepOrWidenR->operands(),
+ vputils::getOpcodeOrIntrinsicID(RepOrWidenR)->second,
+ RepOrWidenR->operands(),
/*Mask=*/nullptr, *RepOrWidenR, {}, DebugLoc::getUnknown(),
RepOrWidenR->getUnderlyingInstr());
Clone->insertBefore(RepOrWidenR);
RepOrWidenR->replaceAllUsesWith(Clone);
- if (isDeadRecipe(*RepOrWidenR))
+ if (vputils::isDeadRecipe(*RepOrWidenR))
RepOrWidenR->eraseFromParent();
}
}
@@ -2159,7 +1974,7 @@ static void simplifyBlends(VPlan &Plan) {
VPValue *DeadMask = Blend->getMask(StartIndex);
Blend->replaceAllUsesWith(NewBlend);
Blend->eraseFromParent();
- recursivelyDeleteDeadRecipes(DeadMask);
+ vputils::recursivelyDeleteDeadRecipes(DeadMask);
/// Simplify BLEND %a, %b, Not(%mask) -> BLEND %b, %a, %mask.
VPValue *NewMask;
@@ -2441,41 +2256,6 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
return true;
}
-/// From the definition of llvm.experimental.get.vector.length,
-/// VPInstruction::ExplicitVectorLength(%AVL) = %AVL when %AVL <= VF.
-bool VPlanTransforms::simplifyKnownEVL(VPlan &Plan, ElementCount VF,
- PredicatedScalarEvolution &PSE) {
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_deep(Plan.getEntry()))) {
- for (VPRecipeBase &R : *VPBB) {
- VPValue *AVL;
- if (!match(&R, m_EVL(m_VPValue(AVL))))
- continue;
-
- const SCEV *AVLSCEV = vputils::getSCEVExprForVPValue(AVL, PSE);
- if (isa<SCEVCouldNotCompute>(AVLSCEV))
- continue;
- ScalarEvolution &SE = *PSE.getSE();
- const SCEV *VFSCEV = SE.getElementCount(AVLSCEV->getType(), VF);
- if (!SE.isKnownPredicate(CmpInst::ICMP_ULE, AVLSCEV, VFSCEV))
- continue;
-
- VPValue *Trunc = VPBuilder(&R).createScalarZExtOrTrunc(
- AVL, Type::getInt32Ty(Plan.getContext()), AVLSCEV->getType(),
- R.getDebugLoc());
- if (Trunc != AVL) {
- auto *TruncR = cast<VPSingleDefRecipe>(Trunc);
- const DataLayout &DL = Plan.getDataLayout();
- if (VPValue *Folded = tryToFoldLiveIns(*TruncR, TruncR->operands(), DL))
- Trunc = Folded;
- }
- R.getVPSingleValue()->replaceAllUsesWith(Trunc);
- return true;
- }
- }
- return false;
-}
-
void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
unsigned BestUF,
PredicatedScalarEvolution &PSE) {
@@ -2503,7 +2283,7 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
RK != RecurKind::AddChainWithSubs)
continue;
- for (VPUser *U : collectUsersRecursively(PhiR))
+ for (VPUser *U : vputils::collectUsersRecursively(PhiR))
if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(U)) {
RecWithFlags->dropPoisonGeneratingFlags();
}
@@ -2533,7 +2313,7 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
// We can extend the list of handled recipes in the future,
// provided we account for the data embedded in them while checking for
// equality or hashing.
- auto C = getOpcodeOrIntrinsicID(Def);
+ auto C = vputils::getOpcodeOrIntrinsicID(Def);
// The issue with (Insert|Extract)Value is that the index of the
// insert/extract is not a proper operand in LLVM IR, and hence also not in
@@ -2549,7 +2329,7 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
/// Hash the underlying data of \p Def.
static unsigned getHashValue(const VPSingleDefRecipe *Def) {
hash_code Result = hash_combine(
- Def->getVPRecipeID(), getOpcodeOrIntrinsicID(Def),
+ Def->getVPRecipeID(), vputils::getOpcodeOrIntrinsicID(Def),
getGEPSourceElementType(Def), Def->getScalarType(),
vputils::isSingleScalar(Def), hash_combine_range(Def->operands()));
if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Def))
@@ -2563,12 +2343,14 @@ struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
/// Check equality of underlying data of \p L and \p R.
static bool isEqual(const VPSingleDefRecipe *L, const VPSingleDefRecipe *R) {
if (L->getVPRecipeID() != R->getVPRecipeID() ||
- getOpcodeOrIntrinsicID(L) != getOpcodeOrIntrinsicID(R) ||
+ vputils::getOpcodeOrIntrinsicID(L) !=
+ vputils::getOpcodeOrIntrinsicID(R) ||
getGEPSourceElementType(L) != getGEPSourceElementType(R) ||
vputils::isSingleScalar(L) != vputils::isSingleScalar(R) ||
!equal(L->operands(), R->operands()))
return false;
- assert(getOpcodeOrIntrinsicID(L) && getOpcodeOrIntrinsicID(R) &&
+ assert(vputils::getOpcodeOrIntrinsicID(L) &&
+ vputils::getOpcodeOrIntrinsicID(R) &&
"must have valid opcode info for both recipes");
if (auto *LFlags = dyn_cast<VPRecipeWithIRFlags>(L))
if (LFlags->hasPredicate() &&
@@ -3055,601 +2837,6 @@ void VPlanTransforms::materializeHeaderMask(
HeaderMask->replaceAllUsesWith(Mask);
}
-template <typename Op0_t, typename Op1_t> struct RemoveMask_match {
- Op0_t In;
- Op1_t &Out;
-
- RemoveMask_match(const Op0_t &In, Op1_t &Out) : In(In), Out(Out) {}
-
- template <typename OpTy> bool match(OpTy *V) const {
- if (m_Specific(In).match(V)) {
- Out = nullptr;
- return true;
- }
- return m_LogicalAnd(m_Specific(In), m_VPValue(Out)).match(V);
- }
-};
-
-/// Match a specific mask \p In, or a combination of it (logical-and In, Out).
-/// Returns the remaining part \p Out if so, or nullptr otherwise.
-template <typename Op0_t, typename Op1_t>
-static inline RemoveMask_match<Op0_t, Op1_t> m_RemoveMask(const Op0_t &In,
- Op1_t &Out) {
- return RemoveMask_match<Op0_t, Op1_t>(In, Out);
-}
-
-static std::optional<Intrinsic::ID> getVPDivRemIntrinsic(Intrinsic::ID IntrID) {
- switch (IntrID) {
- case Intrinsic::masked_udiv:
- return Intrinsic::vp_udiv;
- case Intrinsic::masked_sdiv:
- return Intrinsic::vp_sdiv;
- case Intrinsic::masked_urem:
- return Intrinsic::vp_urem;
- case Intrinsic::masked_srem:
- return Intrinsic::vp_srem;
- default:
- return std::nullopt;
- }
-}
-
-/// Try to optimize a \p CurRecipe masked by \p HeaderMask to a corresponding
-/// EVL-based recipe without the header mask. Returns nullptr if no EVL-based
-/// recipe could be created.
-/// \p HeaderMask Header Mask.
-/// \p CurRecipe Recipe to be transform.
-/// \p EVL The explicit vector length parameter of vector-predication
-/// intrinsics.
-static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
- VPRecipeBase &CurRecipe, VPValue &EVL) {
- VPlan *Plan = CurRecipe.getParent()->getPlan();
- DebugLoc DL = CurRecipe.getDebugLoc();
- VPValue *Addr, *Mask, *EndPtr;
-
- /// Adjust any end pointers so that they point to the end of EVL lanes not VF.
- auto AdjustEndPtr = [&CurRecipe, &EVL](VPValue *EndPtr) {
- auto *EVLEndPtr = cast<VPVectorEndPointerRecipe>(EndPtr)->clone();
- EVLEndPtr->insertBefore(&CurRecipe);
- // Cast EVL (i32) to match the VF operand's type.
- VPValue *EVLAsVF = VPBuilder(EVLEndPtr).createScalarZExtOrTrunc(
- &EVL, EVLEndPtr->getOperand(1)->getScalarType(), EVL.getScalarType(),
- DebugLoc::getUnknown());
- EVLEndPtr->setOperand(1, EVLAsVF);
- return EVLEndPtr;
- };
-
- auto GetVPReverse = [&CurRecipe, &EVL, Plan,
- DL](VPValue *V) -> VPWidenIntrinsicRecipe * {
- if (!V)
- return nullptr;
- auto *Reverse = new VPWidenIntrinsicRecipe(
- Intrinsic::experimental_vp_reverse, {V, Plan->getTrue(), &EVL},
- V->getScalarType(), {}, {}, DL);
- Reverse->insertBefore(&CurRecipe);
- return Reverse;
- };
-
- if (match(&CurRecipe,
- m_MaskedLoad(m_VPValue(Addr), m_RemoveMask(HeaderMask, Mask))))
- return new VPWidenLoadEVLRecipe(cast<VPWidenLoadRecipe>(CurRecipe), Addr,
- EVL, Mask);
-
- if (match(&CurRecipe,
- m_MaskedLoad(m_VPValue(EndPtr),
- m_Reverse(m_RemoveMask(HeaderMask, Mask)))) &&
- match(EndPtr, m_VecEndPtr(m_VPValue(), m_Specific(&Plan->getVF())))) {
- Mask = GetVPReverse(Mask);
- Addr = AdjustEndPtr(EndPtr);
- auto *LoadR = new VPWidenLoadEVLRecipe(cast<VPWidenLoadRecipe>(CurRecipe),
- Addr, EVL, Mask);
- LoadR->insertBefore(&CurRecipe);
- VPValue *Poison = Plan->getPoison(LoadR->getScalarType());
- return new VPWidenIntrinsicRecipe(Intrinsic::vector_splice_left,
- {Poison, LoadR, &EVL},
- LoadR->getScalarType(), {}, {}, DL);
- }
-
- VPValue *Stride;
- if (match(&CurRecipe, m_Intrinsic<Intrinsic::experimental_vp_strided_load>(
- m_VPValue(Addr), m_VPValue(Stride),
- m_RemoveMask(HeaderMask, Mask),
- m_TruncOrSelf(m_Specific(&Plan->getVF()))))) {
- if (!Mask)
- Mask = Plan->getTrue();
- auto *NewLoad = cast<VPWidenMemIntrinsicRecipe>(&CurRecipe)->clone();
- NewLoad->setOperand(2, Mask);
- NewLoad->setOperand(3, &EVL);
- return NewLoad;
- }
-
- VPValue *StoredVal;
- if (match(&CurRecipe, m_MaskedStore(m_VPValue(Addr), m_VPValue(StoredVal),
- m_RemoveMask(HeaderMask, Mask))))
- return new VPWidenStoreEVLRecipe(cast<VPWidenStoreRecipe>(CurRecipe), Addr,
- StoredVal, EVL, Mask);
-
- if (match(&CurRecipe,
- m_MaskedStore(m_VPValue(EndPtr), m_VPValue(StoredVal),
- m_Reverse(m_RemoveMask(HeaderMask, Mask)))) &&
- match(EndPtr, m_VecEndPtr(m_VPValue(), m_Specific(&Plan->getVF())))) {
- Mask = GetVPReverse(Mask);
- Addr = AdjustEndPtr(EndPtr);
- VPValue *Poison = Plan->getPoison(StoredVal->getScalarType());
- auto *SpliceR = new VPWidenIntrinsicRecipe(
- Intrinsic::vector_splice_right, {StoredVal, Poison, &EVL},
- StoredVal->getScalarType(), {}, {}, DL);
- SpliceR->insertBefore(&CurRecipe);
- return new VPWidenStoreEVLRecipe(cast<VPWidenStoreRecipe>(CurRecipe), Addr,
- SpliceR, EVL, Mask);
- }
-
- if (auto *Rdx = dyn_cast<VPReductionRecipe>(&CurRecipe))
- if (Rdx->isConditional() &&
- match(Rdx->getCondOp(), m_RemoveMask(HeaderMask, Mask)))
- return new VPReductionEVLRecipe(*Rdx, EVL, Mask);
-
- if (auto *Interleave = dyn_cast<VPInterleaveRecipe>(&CurRecipe))
- if (Interleave->getMask() &&
- match(Interleave->getMask(), m_RemoveMask(HeaderMask, Mask)))
- return new VPInterleaveEVLRecipe(*Interleave, EVL, Mask);
-
- VPValue *LHS, *RHS;
- if (match(&CurRecipe, m_SelectLike(m_RemoveMask(HeaderMask, Mask),
- m_VPValue(LHS), m_VPValue(RHS))))
- return new VPWidenIntrinsicRecipe(
- Intrinsic::vp_merge, {Mask ? Mask : Plan->getTrue(), LHS, RHS, &EVL},
- LHS->getScalarType(), {}, {}, DL);
-
- if (match(&CurRecipe, m_LastActiveLane(m_Specific(HeaderMask)))) {
- Type *Ty = CurRecipe.getVPSingleValue()->getScalarType();
- VPValue *ZExt =
- VPBuilder(&CurRecipe)
- .createScalarZExtOrTrunc(&EVL, Ty, EVL.getScalarType(), DL);
- return new VPInstruction(
- Instruction::Sub, {ZExt, Plan->getConstantInt(Ty, 1)},
- VPIRFlags::getDefaultFlags(Instruction::Sub), {}, DL);
- }
-
- // lhs | (headermask && rhs) -> vp.merge rhs, true, lhs, evl
- if (match(&CurRecipe,
- m_c_BinaryOr(m_VPValue(LHS),
- m_LogicalAnd(m_Specific(HeaderMask), m_VPValue(RHS)))))
- return new VPWidenIntrinsicRecipe(Intrinsic::vp_merge,
- {RHS, Plan->getTrue(), LHS, &EVL},
- LHS->getScalarType(), {}, {}, DL);
-
- if (auto *IntrR = dyn_cast<VPWidenIntrinsicRecipe>(&CurRecipe))
- if (auto VPID = getVPDivRemIntrinsic(IntrR->getVectorIntrinsicID()))
- if (match(IntrR->getOperand(2), m_RemoveMask(HeaderMask, Mask)))
- return new VPWidenIntrinsicRecipe(*VPID,
- {IntrR->getOperand(0),
- IntrR->getOperand(1),
- Mask ? Mask : Plan->getTrue(), &EVL},
- IntrR->getScalarType(), {}, {}, DL);
-
- return nullptr;
-}
-
-/// Optimize away any EVL-based header masks to VP intrinsic based recipes.
-/// The transforms here need to preserve the original semantics.
-void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) {
- // Find the EVL-based header mask if it exists: icmp ult step-vector, EVL
- VPValue *HeaderMask = nullptr, *EVL = nullptr;
- for (VPRecipeBase &R : *Plan.getVectorLoopRegion()->getEntryBasicBlock()) {
- if (match(&R, m_SpecificICmp(CmpInst::ICMP_ULT, m_StepVector(),
- m_VPValue(EVL))) &&
- match(EVL, m_EVL(m_VPValue()))) {
- HeaderMask = R.getVPSingleValue();
- break;
- }
- }
- if (!HeaderMask)
- return;
-
- SmallVector<VPRecipeBase *> OldRecipes;
- for (VPUser *U : collectUsersRecursively(HeaderMask)) {
- VPRecipeBase *R = cast<VPRecipeBase>(U);
- if (auto *NewR = optimizeMaskToEVL(HeaderMask, *R, *EVL)) {
- NewR->insertBefore(R);
- for (auto [Old, New] :
- zip_equal(R->definedValues(), NewR->definedValues()))
- Old->replaceAllUsesWith(New);
- OldRecipes.push_back(R);
- }
- }
-
- // Replace remaining (HeaderMask && Mask) with vp.merge (True, Mask,
- // False, EVL)
- for (VPUser *U : collectUsersRecursively(HeaderMask)) {
- VPValue *Mask;
- if (match(U, m_LogicalAnd(m_Specific(HeaderMask), m_VPValue(Mask)))) {
- auto *LogicalAnd = cast<VPInstruction>(U);
- auto *Merge = new VPWidenIntrinsicRecipe(
- Intrinsic::vp_merge, {Plan.getTrue(), Mask, Plan.getFalse(), EVL},
- Mask->getScalarType(), {}, {}, LogicalAnd->getDebugLoc());
- Merge->insertBefore(LogicalAnd);
- LogicalAnd->replaceAllUsesWith(Merge);
- OldRecipes.push_back(LogicalAnd);
- }
- }
-
- // Pull out left splices from any elementwise op.
- // binop(splice.left(poison, x, evl), live-in)
- // -> splice.left(poison, binop(x,live-in), evl)
- pullOutPermutations(
- Plan,
- [&EVL](const auto &X) {
- return m_Intrinsic<Intrinsic::vector_splice_left>(m_Poison(), X,
- m_Specific(EVL));
- },
- [&Plan, &EVL](auto *X) {
- return new VPWidenIntrinsicRecipe(
- Intrinsic::vector_splice_left,
- {Plan.getPoison(X->getScalarType()), X, EVL}, X->getScalarType(),
- {}, {}, X->getDebugLoc());
- });
-
- // Fold the following splice patterns:
- // splice.right(splice.left(poison, x, evl), poison, evl) -> x
- // vector.reverse(splice.left(poison, x, evl)) -> vp.reverse(x, true, evl)
- // splice.right(vector.reverse(x), poison, evl) -> vp.reverse(x, true, evl)
- for (VPUser *U : collectUsersRecursively(EVL)) {
- auto *R = cast<VPRecipeBase>(U);
- // Remove potentially dead left splices from the transform above.
- if (match(U, m_Intrinsic<Intrinsic::vector_splice_left>()) &&
- R->getVPSingleValue()->getNumUsers() == 0) {
- OldRecipes.push_back(R);
- continue;
- }
-
- VPValue *X;
- if (match(U, m_Intrinsic<Intrinsic::vector_splice_right>(
- m_Intrinsic<Intrinsic::vector_splice_left>(
- m_Poison(), m_VPValue(X), m_Specific(EVL)),
- m_Poison(), m_Specific(EVL)))) {
- R->getVPSingleValue()->replaceAllUsesWith(X);
- OldRecipes.push_back(R);
- continue;
- }
-
- if (!match(U,
- m_CombineOr(
- m_Reverse(m_Intrinsic<Intrinsic::vector_splice_left>(
- m_Poison(), m_VPValue(X), m_Specific(EVL))),
- m_Intrinsic<Intrinsic::vector_splice_right>(
- m_Reverse(m_VPValue(X)), m_Poison(), m_Specific(EVL)))))
- continue;
-
- auto *VPReverse = new VPWidenIntrinsicRecipe(
- Intrinsic::experimental_vp_reverse, {X, Plan.getTrue(), EVL},
- X->getScalarType(), {}, {}, R->getDebugLoc());
- VPReverse->insertBefore(R);
- R->getVPSingleValue()->replaceAllUsesWith(VPReverse);
- OldRecipes.push_back(R);
- }
-
- for (VPRecipeBase *R : reverse(OldRecipes)) {
- SmallVector<VPValue *> PossiblyDead(R->operands());
- R->eraseFromParent();
- for (VPValue *Op : PossiblyDead)
- recursivelyDeleteDeadRecipes(Op);
- }
-}
-
-/// After replacing the canonical IV with a EVL-based IV, fixup recipes that use
-/// VF to use the EVL instead to avoid incorrect updates on the penultimate
-/// iteration.
-static void fixupVFUsersForEVL(VPlan &Plan, VPValue &EVL) {
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
-
- // EVL is i32 but VF/VFxUF are IdxTy. Convert as needed.
- VPValue *EVLAsIdx =
- VPBuilder::getToInsertAfter(EVL.getDefiningRecipe())
- .createScalarZExtOrTrunc(&EVL, Plan.getVF().getScalarType(),
- EVL.getScalarType(), DebugLoc::getUnknown());
-
- assert(all_of(Plan.getVF().users(),
- [&Plan](VPUser *U) {
- auto IsAllowedUser =
- IsaPred<VPVectorEndPointerRecipe, VPScalarIVStepsRecipe,
- VPWidenIntOrFpInductionRecipe,
- VPWidenMemIntrinsicRecipe>;
- if (match(U, m_Trunc(m_Specific(&Plan.getVF()))))
- return all_of(cast<VPSingleDefRecipe>(U)->users(),
- IsAllowedUser);
- return IsAllowedUser(U);
- }) &&
- "User of VF that we can't transform to EVL.");
- Plan.getVF().replaceUsesWithIf(EVLAsIdx, [](VPUser &U, unsigned Idx) {
- return isa<VPWidenIntOrFpInductionRecipe, VPScalarIVStepsRecipe>(U);
- });
-
- assert(all_of(Plan.getVFxUF().users(),
- match_fn(m_CombineOr(
- m_c_Add(m_Specific(LoopRegion->getCanonicalIV()),
- m_Specific(&Plan.getVFxUF())),
- m_Isa<VPWidenPointerInductionRecipe>()))) &&
- "Only users of VFxUF should be VPWidenPointerInductionRecipe and the "
- "increment of the canonical induction.");
- Plan.getVFxUF().replaceUsesWithIf(EVLAsIdx, [](VPUser &U, unsigned Idx) {
- // Only replace uses in VPWidenPointerInductionRecipe; The increment of the
- // canonical induction must not be updated.
- return isa<VPWidenPointerInductionRecipe>(U);
- });
-
- // Create a scalar phi to track the previous EVL if fixed-order recurrence is
- // contained.
- bool ContainsFORs =
- any_of(Header->phis(), IsaPred<VPFirstOrderRecurrencePHIRecipe>);
- if (ContainsFORs) {
- // TODO: Use VPInstruction::ExplicitVectorLength to get maximum EVL.
- VPValue *MaxEVL = &Plan.getVF();
- // Emit VPScalarCastRecipe in preheader if VF is not a 32 bits integer.
- VPBuilder Builder(LoopRegion->getPreheaderVPBB());
- MaxEVL = Builder.createScalarZExtOrTrunc(
- MaxEVL, Type::getInt32Ty(Plan.getContext()), MaxEVL->getScalarType(),
- DebugLoc::getUnknown());
-
- Builder.setInsertPoint(Header, Header->getFirstNonPhi());
- VPValue *PrevEVL = Builder.createScalarPhi(
- {MaxEVL, &EVL}, DebugLoc::getUnknown(), "prev.evl");
-
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_deep(Plan.getVectorLoopRegion()->getEntry()))) {
- for (VPRecipeBase &R : *VPBB) {
- VPValue *V1, *V2;
- if (!match(&R,
- m_VPInstruction<VPInstruction::FirstOrderRecurrenceSplice>(
- m_VPValue(V1), m_VPValue(V2))))
- continue;
- VPValue *Imm = Plan.getOrAddLiveIn(
- ConstantInt::getSigned(Type::getInt32Ty(Plan.getContext()), -1));
- VPWidenIntrinsicRecipe *VPSplice = new VPWidenIntrinsicRecipe(
- Intrinsic::experimental_vp_splice,
- {V1, V2, Imm, Plan.getTrue(), PrevEVL, &EVL},
- R.getVPSingleValue()->getScalarType(), {}, {}, R.getDebugLoc());
- VPSplice->insertBefore(&R);
- R.getVPSingleValue()->replaceAllUsesWith(VPSplice);
- }
- }
- }
-
- VPValue *HeaderMask = LoopRegion->getHeaderMask();
- if (!HeaderMask)
- return;
-
- // Ensure that any reduction that uses a select to mask off tail lanes does so
- // in the vector loop, not the middle block, since EVL tail folding can have
- // tail elements in the penultimate iteration.
- assert(all_of(*Plan.getMiddleBlock(), [&Plan, HeaderMask](VPRecipeBase &R) {
- if (match(&R, m_ComputeReductionResult(m_Select(m_Specific(HeaderMask),
- m_VPValue(), m_VPValue()))))
- return R.getOperand(0)->getDefiningRecipe()->getRegion() ==
- Plan.getVectorLoopRegion();
- return true;
- }));
-
- // Replace the abstract header mask with a mask equivalent to predicating by
- // EVL: icmp ult step-vector, EVL
- VPRecipeBase *EVLR = EVL.getDefiningRecipe();
- VPBuilder Builder(EVLR->getParent(), std::next(EVLR->getIterator()));
- Type *EVLType = EVL.getScalarType();
- VPValue *EVLMask = Builder.createICmp(
- CmpInst::ICMP_ULT,
- Builder.createNaryOp(VPInstruction::StepVector, {}, EVLType), &EVL);
- HeaderMask->replaceAllUsesWith(EVLMask);
-}
-
-/// Converts a tail folded vector loop region to step by
-/// VPInstruction::ExplicitVectorLength elements instead of VF elements each
-/// iteration.
-///
-/// - Add a VPCurrentIterationPHIRecipe and related recipes to \p Plan and
-/// replaces all uses of the canonical IV except for the canonical IV
-/// increment with a VPCurrentIterationPHIRecipe. The canonical IV is used
-/// only for loop iterations counting after this transformation.
-///
-/// - The header mask is replaced with a header mask based on the EVL.
-///
-/// - Plans with FORs have a new phi added to keep track of the EVL of the
-/// previous iteration, and VPFirstOrderRecurrencePHIRecipes are replaced with
-/// @llvm.vp.splice.
-///
-/// The function uses the following definitions:
-/// %StartV is the canonical induction start value.
-///
-/// The function adds the following recipes:
-///
-/// vector.ph:
-/// ...
-///
-/// vector.body:
-/// ...
-/// %CurrentIter = CURRENT-ITERATION-PHI [ %StartV, %vector.ph ],
-/// [ %NextIter, %vector.body ]
-/// %AVL = phi [ trip-count, %vector.ph ], [ %NextAVL, %vector.body ]
-/// %VPEVL = EXPLICIT-VECTOR-LENGTH %AVL
-/// ...
-/// %OpEVL = cast i32 %VPEVL to IVSize
-/// %NextIter = add IVSize %OpEVL, %CurrentIter
-/// %NextAVL = sub IVSize nuw %AVL, %OpEVL
-/// ...
-///
-/// If MaxSafeElements is provided, the function adds the following recipes:
-/// vector.ph:
-/// ...
-///
-/// vector.body:
-/// ...
-/// %CurrentIter = CURRENT-ITERATION-PHI [ %StartV, %vector.ph ],
-/// [ %NextIter, %vector.body ]
-/// %AVL = phi [ trip-count, %vector.ph ], [ %NextAVL, %vector.body ]
-/// %cmp = cmp ult %AVL, MaxSafeElements
-/// %SAFE_AVL = select %cmp, %AVL, MaxSafeElements
-/// %VPEVL = EXPLICIT-VECTOR-LENGTH %SAFE_AVL
-/// ...
-/// %OpEVL = cast i32 %VPEVL to IVSize
-/// %NextIter = add IVSize %OpEVL, %CurrentIter
-/// %NextAVL = sub IVSize nuw %AVL, %OpEVL
-/// ...
-///
-void VPlanTransforms::addExplicitVectorLength(
- VPlan &Plan, const std::optional<unsigned> &MaxSafeElements) {
- if (Plan.hasScalarVFOnly())
- return;
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
-
- auto *CanonicalIV = LoopRegion->getCanonicalIV();
- auto *CanIVTy = LoopRegion->getCanonicalIVType();
- VPValue *StartV = Plan.getZero(CanIVTy);
- auto *CanonicalIVIncrement = LoopRegion->getOrCreateCanonicalIVIncrement();
-
- // Create the CurrentIteration recipe in the vector loop.
- auto *CurrentIteration =
- new VPCurrentIterationPHIRecipe(StartV, DebugLoc::getUnknown());
- CurrentIteration->insertBefore(*Header, Header->begin());
- VPBuilder Builder(Header, Header->getFirstNonPhi());
- // Create the AVL (application vector length), starting from TC -> 0 in steps
- // of EVL.
- VPPhi *AVLPhi = Builder.createScalarPhi(
- {Plan.getTripCount()}, DebugLoc::getCompilerGenerated(), "avl");
- VPValue *AVL = AVLPhi;
-
- if (MaxSafeElements) {
- // Support for MaxSafeDist for correct loop emission.
- VPValue *AVLSafe = Plan.getConstantInt(CanIVTy, *MaxSafeElements);
- VPValue *Cmp = Builder.createICmp(ICmpInst::ICMP_ULT, AVL, AVLSafe);
- AVL = Builder.createSelect(Cmp, AVL, AVLSafe, DebugLoc::getUnknown(),
- "safe_avl");
- }
- auto *VPEVL = Builder.createNaryOp(VPInstruction::ExplicitVectorLength, AVL,
- DebugLoc::getUnknown(), "evl");
-
- Builder.setInsertPoint(CanonicalIVIncrement);
- VPValue *OpVPEVL = VPEVL;
-
- auto *I32Ty = Type::getInt32Ty(Plan.getContext());
- OpVPEVL = Builder.createScalarZExtOrTrunc(
- OpVPEVL, CanIVTy, I32Ty, CanonicalIVIncrement->getDebugLoc());
-
- auto *NextIter = Builder.createAdd(
- OpVPEVL, CurrentIteration, CanonicalIVIncrement->getDebugLoc(),
- "current.iteration.next", CanonicalIVIncrement->getNoWrapFlags());
- CurrentIteration->addBackedgeValue(NextIter);
-
- VPValue *NextAVL =
- Builder.createSub(AVLPhi, OpVPEVL, DebugLoc::getCompilerGenerated(),
- "avl.next", {/*NUW=*/true, /*NSW=*/false});
- AVLPhi->addIncoming(NextAVL);
-
- fixupVFUsersForEVL(Plan, *VPEVL);
- removeDeadRecipes(Plan);
-
- // Replace all uses of the canonical IV with VPCurrentIterationPHIRecipe
- // except for the canonical IV increment.
- CanonicalIV->replaceUsesWithIf(CurrentIteration,
- [CanonicalIVIncrement](VPUser &U, unsigned) {
- return &U != CanonicalIVIncrement;
- });
- // TODO: support unroll factor > 1.
- Plan.setUF(1);
-}
-
-void VPlanTransforms::convertToVariableLengthStep(VPlan &Plan) {
- // Find the vector loop entry by locating VPCurrentIterationPHIRecipe.
- // There should be only one VPCurrentIteration in the entire plan.
- VPCurrentIterationPHIRecipe *CurrentIteration = nullptr;
-
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
- vp_depth_first_shallow(Plan.getEntry())))
- for (VPRecipeBase &R : VPBB->phis())
- if (auto *PhiR = dyn_cast<VPCurrentIterationPHIRecipe>(&R)) {
- assert(!CurrentIteration &&
- "Found multiple CurrentIteration. Only one expected");
- CurrentIteration = PhiR;
- }
-
- // Early return if it is not variable-length stepping.
- if (!CurrentIteration)
- return;
-
- VPBasicBlock *HeaderVPBB = CurrentIteration->getParent();
- VPValue *CurrentIterationIncr = CurrentIteration->getBackedgeValue();
-
- // Convert CurrentIteration to concrete recipe.
- auto *ScalarR =
- VPBuilder(CurrentIteration)
- .createScalarPhi(
- {CurrentIteration->getStartValue(), CurrentIterationIncr},
- CurrentIteration->getDebugLoc(), "current.iteration.iv");
- CurrentIteration->replaceAllUsesWith(ScalarR);
- CurrentIteration->eraseFromParent();
-
- // Replace CanonicalIVInc with CurrentIteration increment if it exists.
- auto *CanonicalIV = cast<VPPhi>(&*HeaderVPBB->begin());
- if (auto *CanIVInc = findUserOf(
- CanonicalIV, m_c_Add(m_VPValue(), m_Specific(&Plan.getVFxUF())))) {
- cast<VPInstruction>(CanIVInc)->replaceAllUsesWith(CurrentIterationIncr);
- CanIVInc->eraseFromParent();
- }
-}
-
-void VPlanTransforms::convertEVLExitCond(VPlan &Plan) {
- VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
- if (!LoopRegion)
- return;
- VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
- if (Header->empty())
- return;
- // The EVL IV is always at the beginning.
- auto *EVLPhi = dyn_cast<VPCurrentIterationPHIRecipe>(&Header->front());
- if (!EVLPhi)
- return;
-
- // Bail if not an EVL tail folded loop.
- VPValue *AVL;
- if (!match(EVLPhi->getBackedgeValue(),
- m_c_Add(m_ZExtOrSelf(m_EVL(m_VPValue(AVL))), m_Specific(EVLPhi))))
- return;
-
- // The AVL may be capped to a safe distance.
- VPValue *SafeAVL, *UnsafeAVL;
- if (match(AVL,
- m_Select(m_SpecificICmp(CmpInst::ICMP_ULT, m_VPValue(UnsafeAVL),
- m_VPValue(SafeAVL)),
- m_Deferred(UnsafeAVL), m_Deferred(SafeAVL))))
- AVL = UnsafeAVL;
-
- VPValue *AVLNext;
- [[maybe_unused]] bool FoundAVLNext =
- match(AVL, m_VPInstruction<Instruction::PHI>(
- m_Specific(Plan.getTripCount()), m_VPValue(AVLNext)));
- assert(FoundAVLNext && "Didn't find AVL backedge?");
-
- VPBasicBlock *Latch = LoopRegion->getExitingBasicBlock();
- auto *LatchBr = cast<VPInstruction>(Latch->getTerminator());
- if (match(LatchBr, m_BranchOnCond(m_True())))
- return;
-
- VPValue *CanIVInc;
- [[maybe_unused]] bool FoundIncrement = match(
- LatchBr,
- m_BranchOnCond(m_SpecificCmp(CmpInst::ICMP_EQ, m_VPValue(CanIVInc),
- m_Specific(&Plan.getVectorTripCount()))));
- assert(FoundIncrement &&
- match(CanIVInc, m_Add(m_Specific(LoopRegion->getCanonicalIV()),
- m_Specific(&Plan.getVFxUF()))) &&
- "Expected BranchOnCond with ICmp comparing CanIV + VFxUF with vector "
- "trip count");
-
- Type *AVLTy = AVLNext->getScalarType();
- VPBuilder Builder(LatchBr);
- LatchBr->setOperand(
- 0, Builder.createICmp(CmpInst::ICMP_EQ, AVLNext, Plan.getZero(AVLTy)));
-}
-
void VPlanTransforms::replaceSymbolicStrides(
VPlan &Plan, PredicatedScalarEvolution &PSE,
const DenseMap<Value *, const SCEV *> &StridesMap,
@@ -5906,7 +5093,8 @@ static bool canNarrowOps(ArrayRef<VPValue *> Ops, bool IsScalable) {
if (!isa<VPWidenRecipe, VPWidenCastRecipe>(V))
return false;
auto *R = cast<VPRecipeWithIRFlags>(V);
- if (getOpcodeOrIntrinsicID(R) != getOpcodeOrIntrinsicID(WideMember0))
+ if (vputils::getOpcodeOrIntrinsicID(R) !=
+ vputils::getOpcodeOrIntrinsicID(WideMember0))
return false;
if (R->getScalarType() != WideMember0->getScalarType())
return false;
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 85375625d34b5..7ff9c7ad6913e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -14,6 +14,8 @@
#define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
#include "VPlan.h"
+#include "VPlanCFG.h"
+#include "VPlanPatternMatch.h"
#include "VPlanVerifier.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/ScopeExit.h"
@@ -597,6 +599,43 @@ struct VPlanTransforms {
VPCostContext &CostCtx);
};
+/// Removes the permutation pattern \p Perm from any elementwise operations
+/// in the plan, by constructing a new permutation via \p Build.
+/// e.g. binop(perm(x), perm(y)) -> perm(binop(x,y)).
+template <typename Match_t, typename Builder>
+void pullOutPermutations(VPlan &Plan, Match_t Perm, Builder Build) {
+ using namespace VPlanPatternMatch;
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getEntry()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
+ if (!Def || !vputils::isElementwise(Def))
+ continue;
+
+ // At least one of the ops must be a permutation.
+ if (!any_of(Def->operands(), match_fn(Perm(m_VPValue()))))
+ continue;
+
+ // All operands must be permuted or a live in (splat).
+ if (!all_of(
+ Def->operands(),
+ match_fn(m_CombineOr(m_OneUse(Perm(m_VPValue())), m_LiveIn()))))
+ continue;
+
+ VPValue *X;
+ // Remove the inner permutations.
+ for (unsigned I = 0; I < Def->getNumOperands(); I++)
+ if (match(Def->getOperand(I), Perm(m_VPValue(X))))
+ Def->setOperand(I, X);
+
+ VPSingleDefRecipe *Res = Build(Def);
+ Res->insertAfter(Def);
+ Def->replaceUsesWithIf(
+ Res, [&Res](VPUser &U, unsigned _) { return &U != Res; });
+ }
+ }
+}
+
} // namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 03e1ead89c169..996d5d82adaaf 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -12,7 +12,9 @@
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
#include "VPlanPatternMatch.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
@@ -892,3 +894,145 @@ VPValue *VPSCEVExpander::tryToExpand(const SCEV *S) {
return nullptr;
}
}
+
+std::optional<std::pair<bool, unsigned>>
+vputils::getOpcodeOrIntrinsicID(const VPSingleDefRecipe *R) {
+ if (Intrinsic::ID IID = vputils::getIntrinsicID(R))
+ return std::make_pair(true, IID);
+ return TypeSwitch<const VPSingleDefRecipe *,
+ std::optional<std::pair<bool, unsigned>>>(R)
+ .Case<VPInstruction, VPWidenRecipe, VPWidenCastRecipe, VPWidenGEPRecipe,
+ VPReplicateRecipe>(
+ [](auto *I) { return std::make_pair(false, I->getOpcode()); })
+ .Case([](const VPWidenPHIRecipe *I) {
+ return std::make_pair(false, Instruction::PHI);
+ })
+ .Case<VPVectorPointerRecipe, VPPredInstPHIRecipe, VPScalarIVStepsRecipe>(
+ [](auto *I) {
+ // For recipes that do not directly map to LLVM IR instructions,
+ // assign opcodes after the last VPInstruction opcode (which is also
+ // after the last IR Instruction opcode), based on the VPRecipeID.
+ return std::make_pair(false, VPInstruction::OpsEnd + 1 +
+ I->getVPRecipeID());
+ })
+ .Default([](auto *) { return std::nullopt; });
+}
+
+bool vputils::isDeadRecipe(VPRecipeBase &R) {
+ // Do remove conditional assume instructions as their conditions may be
+ // flattened.
+ auto *RepR = dyn_cast<VPReplicateRecipe>(&R);
+ bool IsConditionalAssume = RepR && RepR->isPredicated() &&
+ match(RepR, m_Intrinsic<Intrinsic::assume>());
+ if (IsConditionalAssume)
+ return true;
+
+ if (R.mayHaveSideEffects())
+ return false;
+
+ // Recipe is dead if no user keeps the recipe alive.
+ return all_of(R.definedValues(), [](VPValue *V) { return V->user_empty(); });
+}
+
+void vputils::recursivelyDeleteDeadRecipes(VPValue *V) {
+ SmallVector<VPValue *> WorkList;
+ SmallPtrSet<VPValue *, 8> Seen;
+ WorkList.push_back(V);
+
+ while (!WorkList.empty()) {
+ VPValue *Cur = WorkList.pop_back_val();
+ if (!Seen.insert(Cur).second)
+ continue;
+ VPRecipeBase *R = Cur->getDefiningRecipe();
+ if (!R)
+ continue;
+ if (!isDeadRecipe(*R))
+ continue;
+ append_range(WorkList, R->operands());
+ R->eraseFromParent();
+ }
+}
+
+SmallVector<VPUser *> vputils::collectUsersRecursively(VPValue *V) {
+ SetVector<VPUser *> Users(llvm::from_range, V->users());
+ for (unsigned I = 0; I != Users.size(); ++I) {
+ VPRecipeBase *Cur = cast<VPRecipeBase>(Users[I]);
+ for (VPValue *V : Cur->definedValues())
+ Users.insert_range(V->users());
+ }
+ return Users.takeVector();
+}
+
+VPIRValue *vputils::tryToFoldLiveIns(VPSingleDefRecipe &R,
+ ArrayRef<VPValue *> Operands,
+ const DataLayout &DL) {
+ auto OpcodeOrIID = getOpcodeOrIntrinsicID(&R);
+ if (!OpcodeOrIID)
+ return nullptr;
+
+ SmallVector<Value *, 4> Ops;
+ for (VPValue *Op : Operands) {
+ VPValue *Candidate = Op;
+ match(Op, m_Broadcast(m_VPValue(Candidate)));
+ if (!match(Candidate, m_LiveIn()))
+ return nullptr;
+ Value *V = Candidate->getUnderlyingValue();
+ if (!V)
+ return nullptr;
+ Ops.push_back(V);
+ }
+
+ VPlan &Plan = *R.getParent()->getPlan();
+ auto FoldToIRValue = [&]() -> Value * {
+ InstSimplifyFolder Folder(DL);
+ if (OpcodeOrIID->first) {
+ // VPInstructions store the called intrinsic as last operand.
+ if (isa<VPInstruction>(R))
+ Ops.pop_back();
+
+ auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(&R);
+ return Folder.FoldIntrinsic(OpcodeOrIID->second, Ops, R.getScalarType(),
+ RFlags ? RFlags->getFastMathFlagsOrNone()
+ : FastMathFlags());
+ }
+ unsigned Opcode = OpcodeOrIID->second;
+ if (Instruction::isBinaryOp(Opcode))
+ return Folder.FoldBinOp(static_cast<Instruction::BinaryOps>(Opcode),
+ Ops[0], Ops[1]);
+ if (Instruction::isCast(Opcode))
+ return Folder.FoldCast(static_cast<Instruction::CastOps>(Opcode), Ops[0],
+ R.getVPSingleValue()->getScalarType());
+ switch (Opcode) {
+ case VPInstruction::Not:
+ return Folder.FoldBinOp(Instruction::BinaryOps::Xor, Ops[0],
+ Constant::getAllOnesValue(Ops[0]->getType()));
+ case Instruction::Select:
+ return Folder.FoldSelect(Ops[0], Ops[1], Ops[2]);
+ case Instruction::ICmp:
+ case Instruction::FCmp:
+ return Folder.FoldCmp(cast<VPRecipeWithIRFlags>(R).getPredicate(), Ops[0],
+ Ops[1]);
+ case Instruction::GetElementPtr: {
+ auto &RFlags = cast<VPRecipeWithIRFlags>(R);
+ auto *GEP = cast<GetElementPtrInst>(RFlags.getUnderlyingInstr());
+ return Folder.FoldGEP(GEP->getSourceElementType(), Ops[0],
+ drop_begin(Ops), RFlags.getGEPNoWrapFlags());
+ }
+ case VPInstruction::PtrAdd:
+ case VPInstruction::WidePtrAdd:
+ return Folder.FoldGEP(IntegerType::getInt8Ty(Plan.getContext()), Ops[0],
+ Ops[1],
+ cast<VPRecipeWithIRFlags>(R).getGEPNoWrapFlags());
+ // An extract of a live-in is an extract of a broadcast, so return the
+ // broadcasted element.
+ case Instruction::ExtractElement:
+ assert(!Ops[0]->getType()->isVectorTy() && "Live-ins should be scalar");
+ return Ops[0];
+ }
+ return nullptr;
+ };
+
+ if (Value *V = FoldToIRValue())
+ return Plan.getOrAddLiveIn(V);
+ return nullptr;
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 2980b704ec8da..e2ca2d7a87d84 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -167,6 +167,29 @@ VPInstruction *findComputeReductionResult(VPReductionPHIRecipe *PhiR);
/// Finds the incoming alias-mask within the vector preheader.
VPValue *findIncomingAliasMask(const VPlan &Plan);
+/// Get any instruction opcode or intrinsic ID data embedded in recipe \p R.
+/// Returns an optional pair, where the first element indicates whether it is
+/// an intrinsic ID.
+std::optional<std::pair<bool, unsigned>>
+getOpcodeOrIntrinsicID(const VPSingleDefRecipe *R);
+
+/// Returns true if \p R is dead, i.e. none of its defined values are used and
+/// it has no side effects (with the exception of conditional assumes, which are
+/// considered dead as their conditions may be flattened).
+bool isDeadRecipe(VPRecipeBase &R);
+
+/// Recursively delete \p V and any of its operands that become dead.
+void recursivelyDeleteDeadRecipes(VPValue *V);
+
+/// Collect all users of \p V, looking through recipes that define other values.
+SmallVector<VPUser *> collectUsersRecursively(VPValue *V);
+
+/// Try to fold \p R using InstSimplifyFolder. Will succeed and return a
+/// non-nullptr VPValue for a handled opcode or intrinsic ID if corresponding \p
+/// Operands are foldable live-ins.
+VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R, ArrayRef<VPValue *> Operands,
+ const DataLayout &DL);
+
} // namespace vputils
/// Lightweight SCEV-to-VPlan expander. Converts SCEV expressions into
>From cac1e0f1604aa1a41d0e7f67319961e02423baaf Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 17 Jul 2026 14:39:50 +0100
Subject: [PATCH 2/4] !fixup move pullOutPermutations impl to .cpp
---
.../Transforms/Vectorize/VPlanTransforms.h | 44 ++++++-------------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 36 +++++++++++++++
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 12 +++++
3 files changed, 62 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 7ff9c7ad6913e..6999bc7e84589 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -14,8 +14,8 @@
#define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
#include "VPlan.h"
-#include "VPlanCFG.h"
#include "VPlanPatternMatch.h"
+#include "VPlanUtils.h"
#include "VPlanVerifier.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/ScopeExit.h"
@@ -602,38 +602,22 @@ struct VPlanTransforms {
/// Removes the permutation pattern \p Perm from any elementwise operations
/// in the plan, by constructing a new permutation via \p Build.
/// e.g. binop(perm(x), perm(y)) -> perm(binop(x,y)).
+/// \p Perm is a matcher factory: given a sub-pattern it returns a matcher for
+/// the permutation of that sub-pattern. \p Build creates a new permutation
+/// recipe wrapping the given value.
template <typename Match_t, typename Builder>
void pullOutPermutations(VPlan &Plan, Match_t Perm, Builder Build) {
using namespace VPlanPatternMatch;
- for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
- vp_depth_first_deep(Plan.getEntry()))) {
- for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
- auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
- if (!Def || !vputils::isElementwise(Def))
- continue;
-
- // At least one of the ops must be a permutation.
- if (!any_of(Def->operands(), match_fn(Perm(m_VPValue()))))
- continue;
-
- // All operands must be permuted or a live in (splat).
- if (!all_of(
- Def->operands(),
- match_fn(m_CombineOr(m_OneUse(Perm(m_VPValue())), m_LiveIn()))))
- continue;
-
- VPValue *X;
- // Remove the inner permutations.
- for (unsigned I = 0; I < Def->getNumOperands(); I++)
- if (match(Def->getOperand(I), Perm(m_VPValue(X))))
- Def->setOperand(I, X);
-
- VPSingleDefRecipe *Res = Build(Def);
- Res->insertAfter(Def);
- Def->replaceUsesWithIf(
- Res, [&Res](VPUser &U, unsigned _) { return &U != Res; });
- }
- }
+ // Match \p Op against the permutation, binding the permuted value on success.
+ // If \p OneUseOnly is set, only match single-use permutations.
+ auto MatchPerm = [&Perm](VPValue *Op, bool OneUseOnly) -> VPValue * {
+ VPValue *X;
+ if (OneUseOnly ? match(Op, m_OneUse(Perm(m_VPValue(X))))
+ : match(Op, Perm(m_VPValue(X))))
+ return X;
+ return nullptr;
+ };
+ vputils::pullOutPermutations(Plan, MatchPerm, Build);
}
} // namespace llvm
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 996d5d82adaaf..2179d038abc76 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1036,3 +1036,39 @@ VPIRValue *vputils::tryToFoldLiveIns(VPSingleDefRecipe &R,
return Plan.getOrAddLiveIn(V);
return nullptr;
}
+
+void vputils::pullOutPermutations(
+ VPlan &Plan,
+ function_ref<VPValue *(VPValue *Op, bool OneUseOnly)> MatchPerm,
+ function_ref<VPSingleDefRecipe *(VPSingleDefRecipe *X)> BuildPerm) {
+ for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
+ vp_depth_first_deep(Plan.getEntry()))) {
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB)) {
+ auto *Def = dyn_cast<VPSingleDefRecipe>(&R);
+ if (!Def || !isElementwise(Def))
+ continue;
+
+ // At least one of the ops must be a permutation.
+ if (none_of(Def->operands(), [&MatchPerm](VPValue *Op) {
+ return MatchPerm(Op, /*OneUseOnly=*/false);
+ }))
+ continue;
+
+ // All operands must be a single-use permutation or a live in (splat).
+ if (!all_of(Def->operands(), [&MatchPerm](VPValue *Op) {
+ return MatchPerm(Op, /*OneUseOnly=*/true) || match(Op, m_LiveIn());
+ }))
+ continue;
+
+ // Remove the inner permutations.
+ for (unsigned I = 0, E = Def->getNumOperands(); I != E; ++I)
+ if (VPValue *X = MatchPerm(Def->getOperand(I), /*OneUseOnly=*/false))
+ Def->setOperand(I, X);
+
+ VPSingleDefRecipe *Res = BuildPerm(Def);
+ Res->insertAfter(Def);
+ Def->replaceUsesWithIf(
+ Res, [&Res](VPUser &U, unsigned _) { return &U != Res; });
+ }
+ }
+}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index e2ca2d7a87d84..200ac88751663 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -190,6 +190,18 @@ SmallVector<VPUser *> collectUsersRecursively(VPValue *V);
VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R, ArrayRef<VPValue *> Operands,
const DataLayout &DL);
+/// Removes the permutation pattern matched by \p MatchPerm from any elementwise
+/// operations in \p Plan, by constructing a new permutation via \p BuildPerm.
+/// e.g. binop(perm(x), perm(y)) -> perm(binop(x,y)).
+/// \p MatchPerm should match a permutation of some value and, if \p OneUseOnly
+/// is set, only when that permutation has a single use; on a match it returns
+/// the permuted value, and nullptr otherwise. \p BuildPerm creates a new
+/// permutation recipe wrapping the given value.
+void pullOutPermutations(
+ VPlan &Plan,
+ function_ref<VPValue *(VPValue *Op, bool OneUseOnly)> MatchPerm,
+ function_ref<VPSingleDefRecipe *(VPSingleDefRecipe *X)> BuildPerm);
+
} // namespace vputils
/// Lightweight SCEV-to-VPlan expander. Converts SCEV expressions into
>From 6203fbf9194cbaa6e3665680478c1f3cac8d46d8 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 17 Jul 2026 16:20:17 +0100
Subject: [PATCH 3/4] !fixup address comments, thanks
---
.../Vectorize/VPlanEVLTransforms.cpp | 10 ++---
.../Transforms/Vectorize/VPlanTransforms.cpp | 4 +-
.../Transforms/Vectorize/VPlanTransforms.h | 23 -----------
llvm/lib/Transforms/Vectorize/VPlanUtils.cpp | 14 +++----
llvm/lib/Transforms/Vectorize/VPlanUtils.h | 41 +++++++++++--------
5 files changed, 38 insertions(+), 54 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp
index d1ffa88b7eff6..997ac10cd9c30 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp
@@ -8,7 +8,7 @@
///
/// \file
/// This file implements the VPlan-to-VPlan transforms related to explicit
-/// vector length (EVL) support.
+/// vector length (EVL) tail folding support.
///
//===----------------------------------------------------------------------===//
@@ -282,11 +282,11 @@ void VPlanTransforms::optimizeEVLMasks(VPlan &Plan) {
// Pull out left splices from any elementwise op.
// binop(splice.left(poison, x, evl), live-in)
// -> splice.left(poison, binop(x,live-in), evl)
- pullOutPermutations(
+ vputils::pullOutPermutations(
Plan,
- [&EVL](const auto &X) {
- return m_Intrinsic<Intrinsic::vector_splice_left>(m_Poison(), X,
- m_Specific(EVL));
+ [&EVL](VPValue *&X) {
+ return m_Intrinsic<Intrinsic::vector_splice_left>(
+ m_Poison(), m_VPValue(X), m_Specific(EVL));
},
[&Plan, &EVL](auto *X) {
return new VPWidenIntrinsicRecipe(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 8c518715ac8b0..bd20742b3e600 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1736,8 +1736,8 @@ void VPlanTransforms::simplifyRecipes(VPlan &Plan) {
void VPlanTransforms::simplifyReverses(VPlan &Plan) {
// Pull out reverses from any elementwise op.
// binop(reverse(x), reverse(y)) -> reverse(binop(x,y))
- pullOutPermutations(
- Plan, [](const auto &X) { return m_Reverse(X); },
+ vputils::pullOutPermutations(
+ Plan, [](VPValue *&X) { return m_Reverse(m_VPValue(X)); },
[](auto *X) { return new VPInstruction(VPInstruction::Reverse, X); });
// reverse(reverse(x)) -> x
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
index 6999bc7e84589..85375625d34b5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h
@@ -14,8 +14,6 @@
#define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
#include "VPlan.h"
-#include "VPlanPatternMatch.h"
-#include "VPlanUtils.h"
#include "VPlanVerifier.h"
#include "llvm/ADT/STLFunctionalExtras.h"
#include "llvm/ADT/ScopeExit.h"
@@ -599,27 +597,6 @@ struct VPlanTransforms {
VPCostContext &CostCtx);
};
-/// Removes the permutation pattern \p Perm from any elementwise operations
-/// in the plan, by constructing a new permutation via \p Build.
-/// e.g. binop(perm(x), perm(y)) -> perm(binop(x,y)).
-/// \p Perm is a matcher factory: given a sub-pattern it returns a matcher for
-/// the permutation of that sub-pattern. \p Build creates a new permutation
-/// recipe wrapping the given value.
-template <typename Match_t, typename Builder>
-void pullOutPermutations(VPlan &Plan, Match_t Perm, Builder Build) {
- using namespace VPlanPatternMatch;
- // Match \p Op against the permutation, binding the permuted value on success.
- // If \p OneUseOnly is set, only match single-use permutations.
- auto MatchPerm = [&Perm](VPValue *Op, bool OneUseOnly) -> VPValue * {
- VPValue *X;
- if (OneUseOnly ? match(Op, m_OneUse(Perm(m_VPValue(X))))
- : match(Op, Perm(m_VPValue(X))))
- return X;
- return nullptr;
- };
- vputils::pullOutPermutations(Plan, MatchPerm, Build);
-}
-
} // namespace llvm
#endif // LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
index 2179d038abc76..454b5289e7205 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
@@ -1037,9 +1037,8 @@ VPIRValue *vputils::tryToFoldLiveIns(VPSingleDefRecipe &R,
return nullptr;
}
-void vputils::pullOutPermutations(
- VPlan &Plan,
- function_ref<VPValue *(VPValue *Op, bool OneUseOnly)> MatchPerm,
+void vputils::detail::pullOutPermutationsImpl(
+ VPlan &Plan, function_ref<VPValue *(VPValue *Op)> MatchPerm,
function_ref<VPSingleDefRecipe *(VPSingleDefRecipe *X)> BuildPerm) {
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
vp_depth_first_deep(Plan.getEntry()))) {
@@ -1049,20 +1048,19 @@ void vputils::pullOutPermutations(
continue;
// At least one of the ops must be a permutation.
- if (none_of(Def->operands(), [&MatchPerm](VPValue *Op) {
- return MatchPerm(Op, /*OneUseOnly=*/false);
- }))
+ if (none_of(Def->operands(),
+ [&MatchPerm](VPValue *Op) { return MatchPerm(Op); }))
continue;
// All operands must be a single-use permutation or a live in (splat).
if (!all_of(Def->operands(), [&MatchPerm](VPValue *Op) {
- return MatchPerm(Op, /*OneUseOnly=*/true) || match(Op, m_LiveIn());
+ return (Op->hasOneUse() && MatchPerm(Op)) || match(Op, m_LiveIn());
}))
continue;
// Remove the inner permutations.
for (unsigned I = 0, E = Def->getNumOperands(); I != E; ++I)
- if (VPValue *X = MatchPerm(Def->getOperand(I), /*OneUseOnly=*/false))
+ if (VPValue *X = MatchPerm(Def->getOperand(I)))
Def->setOperand(I, X);
VPSingleDefRecipe *Res = BuildPerm(Def);
diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
index 200ac88751663..6e742e381520a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h
@@ -106,6 +106,12 @@ template <typename Ty> Intrinsic::ID getIntrinsicID(const Ty *R) {
return Intrinsic::not_intrinsic;
}
+/// Get any instruction opcode or intrinsic ID data embedded in recipe \p R.
+/// Returns an optional pair, where the first element indicates whether it is
+/// an intrinsic ID.
+std::optional<std::pair<bool, unsigned>>
+getOpcodeOrIntrinsicID(const VPSingleDefRecipe *R);
+
/// Return a MemoryLocation for \p R with noalias metadata populated from
/// \p R, if the recipe is supported and std::nullopt otherwise. The pointer of
/// the location is conservatively set to nullptr.
@@ -167,12 +173,6 @@ VPInstruction *findComputeReductionResult(VPReductionPHIRecipe *PhiR);
/// Finds the incoming alias-mask within the vector preheader.
VPValue *findIncomingAliasMask(const VPlan &Plan);
-/// Get any instruction opcode or intrinsic ID data embedded in recipe \p R.
-/// Returns an optional pair, where the first element indicates whether it is
-/// an intrinsic ID.
-std::optional<std::pair<bool, unsigned>>
-getOpcodeOrIntrinsicID(const VPSingleDefRecipe *R);
-
/// Returns true if \p R is dead, i.e. none of its defined values are used and
/// it has no side effects (with the exception of conditional assumes, which are
/// considered dead as their conditions may be flattened).
@@ -190,17 +190,26 @@ SmallVector<VPUser *> collectUsersRecursively(VPValue *V);
VPIRValue *tryToFoldLiveIns(VPSingleDefRecipe &R, ArrayRef<VPValue *> Operands,
const DataLayout &DL);
-/// Removes the permutation pattern matched by \p MatchPerm from any elementwise
-/// operations in \p Plan, by constructing a new permutation via \p BuildPerm.
+namespace detail {
+
+/// Template-independent implementation for pullOutPermutations.
+void pullOutPermutationsImpl(
+ VPlan &Plan, function_ref<VPValue *(VPValue *Op)> Perm,
+ function_ref<VPSingleDefRecipe *(VPSingleDefRecipe *X)> Build);
+} // namespace detail
+
+/// Removes the permutation pattern \p Perm from any elementwise operations
+/// in the plan, by constructing a new permutation via \p Build.
/// e.g. binop(perm(x), perm(y)) -> perm(binop(x,y)).
-/// \p MatchPerm should match a permutation of some value and, if \p OneUseOnly
-/// is set, only when that permutation has a single use; on a match it returns
-/// the permuted value, and nullptr otherwise. \p BuildPerm creates a new
-/// permutation recipe wrapping the given value.
-void pullOutPermutations(
- VPlan &Plan,
- function_ref<VPValue *(VPValue *Op, bool OneUseOnly)> MatchPerm,
- function_ref<VPSingleDefRecipe *(VPSingleDefRecipe *X)> BuildPerm);
+template <typename Match_t, typename Builder>
+void pullOutPermutations(VPlan &Plan, Match_t Perm, Builder Build) {
+ // Convert matcher to function returing the matched VPValue.
+ auto MatchPerm = [&Perm](VPValue *Op) -> VPValue * {
+ VPValue *X;
+ return match(Op, Perm(X)) ? X : nullptr;
+ };
+ detail::pullOutPermutationsImpl(Plan, MatchPerm, Build);
+}
} // namespace vputils
>From 96757d26ff54fe79cec8c3c1b23854a39b14dbd9 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo at fhahn.com>
Date: Fri, 17 Jul 2026 17:41:09 +0100
Subject: [PATCH 4/4] !fixup fix formatting
---
llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp
index 997ac10cd9c30..06a93d456a70b 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanEVLTransforms.cpp
@@ -12,12 +12,12 @@
///
//===----------------------------------------------------------------------===//
-#include "VPlanTransforms.h"
#include "LoopVectorizationPlanner.h"
#include "VPlan.h"
#include "VPlanCFG.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
+#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/IR/Intrinsics.h"
More information about the llvm-commits
mailing list