[llvm] [SLP][modularisation][NFC] Move getSameOpcode cluster to SLPCompatibilityAnalysis (3/3) (PR #213609)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 00:02:31 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Madhur Amilkanthwar (madhur13490)
<details>
<summary>Changes</summary>
Move the following out of SLPVectorizer.cpp into
SLPCompatibilityAnalysis.{h,cpp}:
* getSameOpcode
* convertTo
* isAlternateInstruction
* findInstructionWithOpcode
* areCompatibleCmpOps
* isCmpSameOrSwapped
Part of the SLPVectorizer.cpp modularization RFC:
https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
---
Patch is 31.18 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/213609.diff
3 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (-318)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp (+305)
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h (+15)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6cc1bae17bbd1..139dd1335f897 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -590,294 +590,6 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
: TargetTransformInfo::SK_PermuteSingleSrc;
}
-namespace {
-
-std::pair<Instruction *, SmallVector<Value *>>
-convertTo(Instruction *I, const InstructionsState &S) {
- Instruction *SelectedOp = S.getMatchingMainOpOrAltOp(I);
- assert(SelectedOp && "Cannot convert the instruction.");
- if (I->isBinaryOp()) {
- BinOpSameOpcodeHelper Converter(I);
- return std::make_pair(SelectedOp, Converter.getOperand(SelectedOp));
- }
- // Use args() to skip the trailing callee operand in CallInst::operands().
- if (auto *CI = dyn_cast<CallInst>(I))
- return std::make_pair(SelectedOp, SmallVector<Value *>(CI->args()));
- return std::make_pair(SelectedOp, SmallVector<Value *>(I->operands()));
-}
-
-} // end anonymous namespace
-
-static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
- const TargetLibraryInfo &TLI);
-
-/// Find an instruction with a specific opcode in VL.
-/// \param VL Array of values to search through. Must contain only Instructions
-/// and PoisonValues.
-/// \param Opcode The instruction opcode to search for
-/// \returns
-/// - The first instruction found with matching opcode
-/// - nullptr if no matching instruction is found
-static Instruction *findInstructionWithOpcode(ArrayRef<Value *> VL,
- unsigned Opcode) {
- for (Value *V : VL) {
- if (isa<PoisonValue>(V))
- continue;
- assert(isa<Instruction>(V) && "Only accepts PoisonValue and Instruction.");
- auto *Inst = cast<Instruction>(V);
- if (Inst->getOpcode() == Opcode)
- return Inst;
- }
- return nullptr;
-}
-
-/// Checks if the provided operands of 2 cmp instructions are compatible, i.e.
-/// compatible instructions or constants, or just some other regular values.
-static bool areCompatibleCmpOps(Value *BaseOp0, Value *BaseOp1, Value *Op0,
- Value *Op1, const TargetLibraryInfo &TLI) {
- return (isConstant(BaseOp0) && isConstant(Op0)) ||
- (isConstant(BaseOp1) && isConstant(Op1)) ||
- (!isa<Instruction>(BaseOp0) && !isa<Instruction>(Op0) &&
- !isa<Instruction>(BaseOp1) && !isa<Instruction>(Op1)) ||
- BaseOp0 == Op0 || BaseOp1 == Op1 ||
- getSameOpcode({BaseOp0, Op0}, TLI) ||
- getSameOpcode({BaseOp1, Op1}, TLI);
-}
-
-/// \returns true if a compare instruction \p CI has similar "look" and
-/// same predicate as \p BaseCI, "as is" or with its operands and predicate
-/// swapped, false otherwise.
-static bool isCmpSameOrSwapped(const CmpInst *BaseCI, const CmpInst *CI,
- const TargetLibraryInfo &TLI) {
- assert(BaseCI->getOperand(0)->getType() == CI->getOperand(0)->getType() &&
- "Assessing comparisons of different types?");
- CmpInst::Predicate BasePred = BaseCI->getPredicate();
- CmpInst::Predicate Pred = CI->getPredicate();
- CmpInst::Predicate SwappedPred = CmpInst::getSwappedPredicate(Pred);
-
- Value *BaseOp0 = BaseCI->getOperand(0);
- Value *BaseOp1 = BaseCI->getOperand(1);
- Value *Op0 = CI->getOperand(0);
- Value *Op1 = CI->getOperand(1);
-
- return (BasePred == Pred &&
- areCompatibleCmpOps(BaseOp0, BaseOp1, Op0, Op1, TLI)) ||
- (BasePred == SwappedPred &&
- areCompatibleCmpOps(BaseOp0, BaseOp1, Op1, Op0, TLI));
-}
-
-/// \returns analysis of the Instructions in \p VL described in
-/// InstructionsState, the Opcode that we suppose the whole list
-/// could be vectorized even if its structure is diverse.
-static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
- const TargetLibraryInfo &TLI) {
- // Make sure these are all Instructions.
- if (!all_of(VL, IsaPred<Instruction, PoisonValue>))
- return InstructionsState::invalid();
-
- auto *It = find_if(VL, IsaPred<Instruction>);
- if (It == VL.end())
- return InstructionsState::invalid();
-
- Instruction *MainOp = cast<Instruction>(*It);
- unsigned InstCnt = std::count_if(It, VL.end(), IsaPred<Instruction>);
- if ((VL.size() > 2 && !isa<PHINode>(MainOp) && InstCnt < VL.size() / 2) ||
- (VL.size() == 2 && InstCnt < 2))
- return InstructionsState::invalid();
-
- bool IsCastOp = isa<CastInst>(MainOp);
- bool IsBinOp = isa<BinaryOperator>(MainOp);
- bool IsCmpOp = isa<CmpInst>(MainOp);
- CmpInst::Predicate BasePred = IsCmpOp ? cast<CmpInst>(MainOp)->getPredicate()
- : CmpInst::BAD_ICMP_PREDICATE;
- Instruction *AltOp = MainOp;
- unsigned Opcode = MainOp->getOpcode();
- unsigned AltOpcode = Opcode;
-
- BinOpSameOpcodeHelper BinOpHelper(MainOp);
- bool SwappedPredsCompatible = IsCmpOp && [&]() {
- SetVector<unsigned> UniquePreds, UniqueNonSwappedPreds;
- UniquePreds.insert(BasePred);
- UniqueNonSwappedPreds.insert(BasePred);
- for (Value *V : VL) {
- auto *I = dyn_cast<CmpInst>(V);
- if (!I)
- return false;
- CmpInst::Predicate CurrentPred = I->getPredicate();
- CmpInst::Predicate SwappedCurrentPred =
- CmpInst::getSwappedPredicate(CurrentPred);
- UniqueNonSwappedPreds.insert(CurrentPred);
- if (!UniquePreds.contains(CurrentPred) &&
- !UniquePreds.contains(SwappedCurrentPred))
- UniquePreds.insert(CurrentPred);
- }
- // Total number of predicates > 2, but if consider swapped predicates
- // compatible only 2, consider swappable predicates as compatible opcodes,
- // not alternate.
- return UniqueNonSwappedPreds.size() > 2 && UniquePreds.size() == 2;
- }();
- // Check for one alternate opcode from another BinaryOperator.
- // TODO - generalize to support all operators (types, calls etc.).
- Intrinsic::ID BaseID = 0;
- SmallVector<VFInfo> BaseMappings;
- if (auto *CallBase = dyn_cast<CallInst>(MainOp)) {
- BaseID = getVectorIntrinsicIDForCall(CallBase, &TLI);
- BaseMappings = VFDatabase(*CallBase).getMappings(*CallBase);
- if (!isTriviallyVectorizable(BaseID) && BaseMappings.empty())
- return InstructionsState::invalid();
- }
- bool AnyPoison = InstCnt != VL.size();
- // Check MainOp too to be sure that it matches the requirements for the
- // instructions.
- for (Value *V : iterator_range(It, VL.end())) {
- auto *I = dyn_cast<Instruction>(V);
- if (!I)
- continue;
-
- // Cannot combine poison and divisions.
- // TODO: do some smart analysis of the CallInsts to exclude divide-like
- // intrinsics/functions only.
- if (AnyPoison && (I->isIntDivRem() || I->isFPDivRem() || isa<CallInst>(I)))
- return InstructionsState::invalid();
- unsigned InstOpcode = I->getOpcode();
- if (IsBinOp && isa<BinaryOperator>(I)) {
- if (BinOpHelper.add(I))
- continue;
- } else if (IsCastOp && isa<CastInst>(I)) {
- Value *Op0 = MainOp->getOperand(0);
- Type *Ty0 = Op0->getType();
- Value *Op1 = I->getOperand(0);
- Type *Ty1 = Op1->getType();
- if (Ty0 == Ty1) {
- if (InstOpcode == Opcode || InstOpcode == AltOpcode)
- continue;
- if (Opcode == AltOpcode) {
- assert(isValidForAlternation(Opcode) &&
- isValidForAlternation(InstOpcode) &&
- "Cast isn't safe for alternation, logic needs to be updated!");
- AltOpcode = InstOpcode;
- AltOp = I;
- continue;
- }
- }
- } else if (auto *Inst = dyn_cast<CmpInst>(I); Inst && IsCmpOp) {
- auto *BaseInst = cast<CmpInst>(MainOp);
- Type *Ty0 = BaseInst->getOperand(0)->getType();
- Type *Ty1 = Inst->getOperand(0)->getType();
- if (Ty0 == Ty1) {
- assert(InstOpcode == Opcode && "Expected same CmpInst opcode.");
- assert(InstOpcode == AltOpcode &&
- "Alternate instructions are only supported by BinaryOperator "
- "and CastInst.");
- // Check for compatible operands. If the corresponding operands are not
- // compatible - need to perform alternate vectorization.
- CmpInst::Predicate CurrentPred = Inst->getPredicate();
- CmpInst::Predicate SwappedCurrentPred =
- CmpInst::getSwappedPredicate(CurrentPred);
-
- if ((VL.size() == 2 || SwappedPredsCompatible) &&
- (BasePred == CurrentPred || BasePred == SwappedCurrentPred))
- continue;
-
- if (isCmpSameOrSwapped(BaseInst, Inst, TLI))
- continue;
- auto *AltInst = cast<CmpInst>(AltOp);
- if (MainOp != AltOp) {
- if (isCmpSameOrSwapped(AltInst, Inst, TLI))
- continue;
- } else if (BasePred != CurrentPred) {
- assert(
- isValidForAlternation(InstOpcode) &&
- "CmpInst isn't safe for alternation, logic needs to be updated!");
- AltOp = I;
- continue;
- }
- CmpInst::Predicate AltPred = AltInst->getPredicate();
- if (BasePred == CurrentPred || BasePred == SwappedCurrentPred ||
- AltPred == CurrentPred || AltPred == SwappedCurrentPred)
- continue;
- }
- } else if (InstOpcode == Opcode) {
- assert(InstOpcode == AltOpcode &&
- "Alternate instructions are only supported by BinaryOperator and "
- "CastInst.");
- if (auto *Gep = dyn_cast<GetElementPtrInst>(I)) {
- if (Gep->getNumOperands() != 2 ||
- Gep->getOperand(0)->getType() != MainOp->getOperand(0)->getType())
- return InstructionsState::invalid();
- } else if (auto *EI = dyn_cast<ExtractElementInst>(I)) {
- if (!isVectorLikeInstWithConstOps(EI))
- return InstructionsState::invalid();
- } else if (auto *LI = dyn_cast<LoadInst>(I)) {
- auto *BaseLI = cast<LoadInst>(MainOp);
- if (!LI->isSimple() || !BaseLI->isSimple())
- return InstructionsState::invalid();
- } else if (auto *Call = dyn_cast<CallInst>(I)) {
- auto *CallBase = cast<CallInst>(MainOp);
- Intrinsic::ID ID = getVectorIntrinsicIDForCall(Call, &TLI);
- Intrinsic::ID Equivalent = isEquivalentIntrinsicID(ID, BaseID);
- if (Call->getCalledFunction() != CallBase->getCalledFunction() &&
- isEquivalentIntrinsicID(Equivalent, Intrinsic::fmuladd) ==
- Intrinsic::not_intrinsic)
- return InstructionsState::invalid();
- if (Call->hasOperandBundles() &&
- (!CallBase->hasOperandBundles() ||
- !std::equal(Call->op_begin() + Call->getBundleOperandsStartIndex(),
- Call->op_begin() + Call->getBundleOperandsEndIndex(),
- CallBase->op_begin() +
- CallBase->getBundleOperandsStartIndex())))
- return InstructionsState::invalid();
- if (ID != BaseID && Equivalent == Intrinsic::not_intrinsic)
- return InstructionsState::invalid();
- if (!ID) {
- SmallVector<VFInfo> Mappings = VFDatabase(*Call).getMappings(*Call);
- if (Mappings.size() != BaseMappings.size() ||
- Mappings.front().ISA != BaseMappings.front().ISA ||
- Mappings.front().ScalarName != BaseMappings.front().ScalarName ||
- Mappings.front().VectorName != BaseMappings.front().VectorName ||
- Mappings.front().Shape.VF != BaseMappings.front().Shape.VF ||
- Mappings.front().Shape.Parameters !=
- BaseMappings.front().Shape.Parameters)
- return InstructionsState::invalid();
- }
- }
- continue;
- }
- return InstructionsState::invalid();
- }
-
- if (IsBinOp) {
- if (!BinOpHelper.hasDefinedMainOpcode() ||
- !BinOpHelper.hasDefinedAltOpcode())
- return InstructionsState::invalid();
- MainOp = findInstructionWithOpcode(VL, BinOpHelper.getMainOpcode());
- assert(MainOp && "Cannot find MainOp with Opcode from BinOpHelper.");
- AltOp = findInstructionWithOpcode(VL, BinOpHelper.getAltOpcode());
- assert(AltOp && "Cannot find AltOp with Opcode from BinOpHelper.");
- } else if (auto *CB = dyn_cast<CallInst>(MainOp);
- CB &&
- getVectorIntrinsicIDForCall(CB, &TLI) == Intrinsic::fmuladd) {
- // fma and fmuladd share a single vector fma node; use the fma as the
- // representative so the fused form is not weakened to fmuladd.
- auto *It = find_if(VL, [&](Value *V) {
- auto *CI = dyn_cast<CallInst>(V);
- return CI && getVectorIntrinsicIDForCall(CI, &TLI) == Intrinsic::fma;
- });
- if (It != VL.end())
- MainOp = AltOp = cast<Instruction>(*It);
- }
- assert((MainOp == AltOp || !allSameOpcode(VL)) &&
- "Incorrect implementation of allSameOpcode.");
- InstructionsState S(MainOp, AltOp);
- assert(all_of(VL,
- [&](Value *V) {
- return isa<PoisonValue>(V) ||
- S.getMatchingMainOpOrAltOp(cast<Instruction>(V));
- }) &&
- "Invalid InstructionsState.");
- return S;
-}
-
/// Returns true if widened type of \p Ty elements with size \p Sz represents
/// full vector type, i.e. adding extra element results in extra parts upon type
/// legalization.
@@ -7507,12 +7219,6 @@ static bool areTwoInsertFromSameBuildVector(
return false;
}
-/// Checks if the specified instruction \p I is an alternate operation for
-/// the given \p MainOp and \p AltOp instructions.
-static bool isAlternateInstruction(Instruction *I, Instruction *MainOp,
- Instruction *AltOp,
- const TargetLibraryInfo &TLI);
-
std::optional<BoUpSLP::OrdersType>
BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
bool IgnoreReorder) {
@@ -13706,30 +13412,6 @@ static bool isMainInstruction(Instruction *I, Instruction *MainOp,
return InstructionsState(MainOp, AltOp).getMatchingMainOpOrAltOp(I) == MainOp;
}
-static bool isAlternateInstruction(Instruction *I, Instruction *MainOp,
- Instruction *AltOp,
- const TargetLibraryInfo &TLI) {
- if (auto *MainCI = dyn_cast<CmpInst>(MainOp)) {
- auto *AltCI = cast<CmpInst>(AltOp);
- CmpInst::Predicate MainP = MainCI->getPredicate();
- [[maybe_unused]] CmpInst::Predicate AltP = AltCI->getPredicate();
- assert(MainP != AltP && "Expected different main/alternate predicates.");
- auto *CI = cast<CmpInst>(I);
- if (isCmpSameOrSwapped(MainCI, CI, TLI))
- return false;
- if (isCmpSameOrSwapped(AltCI, CI, TLI))
- return true;
- CmpInst::Predicate P = CI->getPredicate();
- CmpInst::Predicate SwappedP = CmpInst::getSwappedPredicate(P);
-
- assert((MainP == P || AltP == P || MainP == SwappedP || AltP == SwappedP) &&
- "CmpInst expected to match either main or alternate predicate or "
- "their swap.");
- return MainP != P && MainP != SwappedP;
- }
- return InstructionsState(MainOp, AltOp).getMatchingMainOpOrAltOp(I) == AltOp;
-}
-
TTI::OperandValueInfo BoUpSLP::getOperandInfo(ArrayRef<Value *> Ops) const {
assert(!Ops.empty());
const auto *Op0 = Ops.front();
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
index 54cb526a10eab..85f34eca235e5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
@@ -10,18 +10,23 @@
#include "SLPUtils.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
+#include <algorithm>
#include <array>
#include <cassert>
#include <utility>
@@ -469,4 +474,304 @@ bool InstructionsState::isNonSchedulable(Value *V) const {
doesNotNeedToBeScheduled(V);
}
+/// Find an instruction with a specific opcode in VL.
+/// \param VL Array of values to search through. Must contain only Instructions
+/// and PoisonValues.
+/// \param Opcode The instruction opcode to search for
+/// \returns
+/// - The first instruction found with matching opcode
+/// - nullptr if no matching instruction is found
+static Instruction *findInstructionWithOpcode(ArrayRef<Value *> VL,
+ unsigned Opcode) {
+ for (Value *V : VL) {
+ if (isa<PoisonValue>(V))
+ continue;
+ assert(isa<Instruction>(V) && "Only accepts PoisonValue and Instruction.");
+ auto *Inst = cast<Instruction>(V);
+ if (Inst->getOpcode() == Opcode)
+ return Inst;
+ }
+ return nullptr;
+}
+
+/// Checks if the provided operands of 2 cmp instructions are compatible, i.e.
+/// compatible instructions or constants, or just some other regular values.
+static bool areCompatibleCmpOps(Value *BaseOp0, Value *BaseOp1, Value *Op0,
+ Value *Op1, const TargetLibraryInfo &TLI) {
+ return (isConstant(BaseOp0) && isConstant(Op0)) ||
+ (isConstant(BaseOp1) && isConstant(Op1)) ||
+ (!isa<Instruction>(BaseOp0) && !isa<Instruction>(Op0) &&
+ !isa<Instruction>(BaseOp1) && !isa<Instruction>(Op1)) ||
+ BaseOp0 == Op0 || BaseOp1 == Op1 ||
+ getSameOpcode({BaseOp0, Op0}, TLI) ||
+ getSameOpcode({BaseOp1, Op1}, TLI);
+}
+
+/// \returns true if a compare instruction \p CI has similar "look" and
+/// same predicate as \p BaseCI, "as is" or with its operands and predicate
+/// swapped, false otherwise.
+static bool isCmpSameOrSwapped(const CmpInst *BaseCI, const CmpInst *CI,
+ const TargetLibraryInfo &TLI) {
+ assert(BaseCI->getOperand(0)->getType() == CI->getOperand(0)->getType() &&
+ "Assessing comparisons of different types?");
+ CmpInst::Predicate BasePred = BaseCI->getPredicate();
+ CmpInst::Predicate Pred = CI->getPredicate();
+ CmpInst::Predicate SwappedPred = CmpInst::getSwappedPredicate(Pred);
+
+ Value *BaseOp0 = BaseCI->getOperand(0);
+ Value *BaseOp1 = BaseCI->getOperand(1);
+ Value *Op0 = CI->getOperand(0);
+ Value *Op1 = CI->getOperand(1);
+
+ return (BasePred == Pred &&
+ areCompatibleCmpOps(BaseOp0, BaseOp1, Op0, Op1, TLI)) ||
+ (BasePred == SwappedPred &&
+ areCompatibleCmpOps(BaseOp0, BaseOp1, Op1, Op0, TLI));
+}
+
+InstructionsState getSameOpcode(ArrayRef<Value *> VL,
+ const TargetLibraryInfo &TLI) {
+ // Make sure these are all Instructions.
+ if (!all_of(VL, IsaPred<Instruction, PoisonValue>))
+ return InstructionsState::invalid();
+
+ auto *It = find_if(VL, IsaPred<Instruction>);
+ if (It == VL.end())
+ return InstructionsState::invalid();
+
+ Instruction *MainOp = cast<Instruction>(*It);
+ unsigned InstCnt = std::count_if(It, VL.end(), IsaPred<Instruction>);
+ if ((VL.size() > 2 && !isa<PHINode>(MainOp) && InstCnt < VL.size() / 2) ||
+ (VL.size() == 2 && InstCnt < 2))
+ return InstructionsState::invalid();
+
+ bool IsCastOp = isa<CastInst>(MainOp);
+ bool IsBinOp = isa<BinaryOperator>(MainOp);
+ bool IsCmpOp = isa<CmpInst>(MainOp);
+ CmpInst::Predicate BasePred = IsCmpOp ? cast<CmpInst>(MainOp)->getPredicate()
+ : CmpInst::BAD_ICMP_PREDICATE;
+ Instruction *AltOp = MainOp;
+ unsigned Opcode = MainOp->getOpcode();
+ unsigned AltOpcode = Opcode;
+
+ BinOpSameOpcodeHelper BinOpHelper(MainOp);
+ bool SwappedPredsCompatible = IsCmpOp && [&]() {
+ SetVector<unsigned> UniquePreds, UniqueNonSwappedPreds;
+ UniquePreds.insert(BasePred);
+ UniqueNonSwappedPreds.insert(BasePred);
+ for (Value *V : VL) {
+ auto *I = dyn_cast<CmpInst>(V);
+ ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/213609
More information about the llvm-commits
mailing list