[llvm] [SLP][modularisation][NFC] Move getSameOpcode cluster to SLPCompatibilityAnalysis (3/3) (PR #213609)
Madhur Amilkanthwar via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 00:01:41 PDT 2026
https://github.com/madhur13490 created https://github.com/llvm/llvm-project/pull/213609
Move the following out of SLPVectorizer.cpp into
SLPCompatibilityAnalysis.{h,cpp}:
* getSameOpcode
* convertTo
* isAlternateInstruction
* findInstructionWithOpcode
* areCompatibleCmpOps
* isCmpSameOrSwapped
Part of the SLPVectorizer.cpp modularization RFC:
https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
>From e71f060fb60b3329afb4ca6a41b88042de28d323 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Sun, 2 Aug 2026 23:14:32 -0700
Subject: [PATCH] [SLP][modularisation][NFC] Move getSameOpcode cluster to
SLPCompatibilityAnalysis
Move the following out of SLPVectorizer.cpp into
SLPCompatibilityAnalysis.{h,cpp}:
getSameOpcode
convertTo
isAlternateInstruction
findInstructionWithOpcode
areCompatibleCmpOps
isCmpSameOrSwapped
Part of the SLPVectorizer.cpp modularization RFC:
https://discourse.llvm.org/t/modularizing-slpvectorizer-cpp/90922
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 318 ------------------
.../SLPCompatibilityAnalysis.cpp | 305 +++++++++++++++++
.../SLPVectorizer/SLPCompatibilityAnalysis.h | 15 +
3 files changed, 320 insertions(+), 318 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6cc1bae17bbd1..139dd1335f897 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -590,294 +590,6 @@ isFixedVectorShuffle(ArrayRef<Value *> VL, SmallVectorImpl<int> &Mask,
: TargetTransformInfo::SK_PermuteSingleSrc;
}
-namespace {
-
-std::pair<Instruction *, SmallVector<Value *>>
-convertTo(Instruction *I, const InstructionsState &S) {
- Instruction *SelectedOp = S.getMatchingMainOpOrAltOp(I);
- assert(SelectedOp && "Cannot convert the instruction.");
- if (I->isBinaryOp()) {
- BinOpSameOpcodeHelper Converter(I);
- return std::make_pair(SelectedOp, Converter.getOperand(SelectedOp));
- }
- // Use args() to skip the trailing callee operand in CallInst::operands().
- if (auto *CI = dyn_cast<CallInst>(I))
- return std::make_pair(SelectedOp, SmallVector<Value *>(CI->args()));
- return std::make_pair(SelectedOp, SmallVector<Value *>(I->operands()));
-}
-
-} // end anonymous namespace
-
-static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
- const TargetLibraryInfo &TLI);
-
-/// Find an instruction with a specific opcode in VL.
-/// \param VL Array of values to search through. Must contain only Instructions
-/// and PoisonValues.
-/// \param Opcode The instruction opcode to search for
-/// \returns
-/// - The first instruction found with matching opcode
-/// - nullptr if no matching instruction is found
-static Instruction *findInstructionWithOpcode(ArrayRef<Value *> VL,
- unsigned Opcode) {
- for (Value *V : VL) {
- if (isa<PoisonValue>(V))
- continue;
- assert(isa<Instruction>(V) && "Only accepts PoisonValue and Instruction.");
- auto *Inst = cast<Instruction>(V);
- if (Inst->getOpcode() == Opcode)
- return Inst;
- }
- return nullptr;
-}
-
-/// Checks if the provided operands of 2 cmp instructions are compatible, i.e.
-/// compatible instructions or constants, or just some other regular values.
-static bool areCompatibleCmpOps(Value *BaseOp0, Value *BaseOp1, Value *Op0,
- Value *Op1, const TargetLibraryInfo &TLI) {
- return (isConstant(BaseOp0) && isConstant(Op0)) ||
- (isConstant(BaseOp1) && isConstant(Op1)) ||
- (!isa<Instruction>(BaseOp0) && !isa<Instruction>(Op0) &&
- !isa<Instruction>(BaseOp1) && !isa<Instruction>(Op1)) ||
- BaseOp0 == Op0 || BaseOp1 == Op1 ||
- getSameOpcode({BaseOp0, Op0}, TLI) ||
- getSameOpcode({BaseOp1, Op1}, TLI);
-}
-
-/// \returns true if a compare instruction \p CI has similar "look" and
-/// same predicate as \p BaseCI, "as is" or with its operands and predicate
-/// swapped, false otherwise.
-static bool isCmpSameOrSwapped(const CmpInst *BaseCI, const CmpInst *CI,
- const TargetLibraryInfo &TLI) {
- assert(BaseCI->getOperand(0)->getType() == CI->getOperand(0)->getType() &&
- "Assessing comparisons of different types?");
- CmpInst::Predicate BasePred = BaseCI->getPredicate();
- CmpInst::Predicate Pred = CI->getPredicate();
- CmpInst::Predicate SwappedPred = CmpInst::getSwappedPredicate(Pred);
-
- Value *BaseOp0 = BaseCI->getOperand(0);
- Value *BaseOp1 = BaseCI->getOperand(1);
- Value *Op0 = CI->getOperand(0);
- Value *Op1 = CI->getOperand(1);
-
- return (BasePred == Pred &&
- areCompatibleCmpOps(BaseOp0, BaseOp1, Op0, Op1, TLI)) ||
- (BasePred == SwappedPred &&
- areCompatibleCmpOps(BaseOp0, BaseOp1, Op1, Op0, TLI));
-}
-
-/// \returns analysis of the Instructions in \p VL described in
-/// InstructionsState, the Opcode that we suppose the whole list
-/// could be vectorized even if its structure is diverse.
-static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
- const TargetLibraryInfo &TLI) {
- // Make sure these are all Instructions.
- if (!all_of(VL, IsaPred<Instruction, PoisonValue>))
- return InstructionsState::invalid();
-
- auto *It = find_if(VL, IsaPred<Instruction>);
- if (It == VL.end())
- return InstructionsState::invalid();
-
- Instruction *MainOp = cast<Instruction>(*It);
- unsigned InstCnt = std::count_if(It, VL.end(), IsaPred<Instruction>);
- if ((VL.size() > 2 && !isa<PHINode>(MainOp) && InstCnt < VL.size() / 2) ||
- (VL.size() == 2 && InstCnt < 2))
- return InstructionsState::invalid();
-
- bool IsCastOp = isa<CastInst>(MainOp);
- bool IsBinOp = isa<BinaryOperator>(MainOp);
- bool IsCmpOp = isa<CmpInst>(MainOp);
- CmpInst::Predicate BasePred = IsCmpOp ? cast<CmpInst>(MainOp)->getPredicate()
- : CmpInst::BAD_ICMP_PREDICATE;
- Instruction *AltOp = MainOp;
- unsigned Opcode = MainOp->getOpcode();
- unsigned AltOpcode = Opcode;
-
- BinOpSameOpcodeHelper BinOpHelper(MainOp);
- bool SwappedPredsCompatible = IsCmpOp && [&]() {
- SetVector<unsigned> UniquePreds, UniqueNonSwappedPreds;
- UniquePreds.insert(BasePred);
- UniqueNonSwappedPreds.insert(BasePred);
- for (Value *V : VL) {
- auto *I = dyn_cast<CmpInst>(V);
- if (!I)
- return false;
- CmpInst::Predicate CurrentPred = I->getPredicate();
- CmpInst::Predicate SwappedCurrentPred =
- CmpInst::getSwappedPredicate(CurrentPred);
- UniqueNonSwappedPreds.insert(CurrentPred);
- if (!UniquePreds.contains(CurrentPred) &&
- !UniquePreds.contains(SwappedCurrentPred))
- UniquePreds.insert(CurrentPred);
- }
- // Total number of predicates > 2, but if consider swapped predicates
- // compatible only 2, consider swappable predicates as compatible opcodes,
- // not alternate.
- return UniqueNonSwappedPreds.size() > 2 && UniquePreds.size() == 2;
- }();
- // Check for one alternate opcode from another BinaryOperator.
- // TODO - generalize to support all operators (types, calls etc.).
- Intrinsic::ID BaseID = 0;
- SmallVector<VFInfo> BaseMappings;
- if (auto *CallBase = dyn_cast<CallInst>(MainOp)) {
- BaseID = getVectorIntrinsicIDForCall(CallBase, &TLI);
- BaseMappings = VFDatabase(*CallBase).getMappings(*CallBase);
- if (!isTriviallyVectorizable(BaseID) && BaseMappings.empty())
- return InstructionsState::invalid();
- }
- bool AnyPoison = InstCnt != VL.size();
- // Check MainOp too to be sure that it matches the requirements for the
- // instructions.
- for (Value *V : iterator_range(It, VL.end())) {
- auto *I = dyn_cast<Instruction>(V);
- if (!I)
- continue;
-
- // Cannot combine poison and divisions.
- // TODO: do some smart analysis of the CallInsts to exclude divide-like
- // intrinsics/functions only.
- if (AnyPoison && (I->isIntDivRem() || I->isFPDivRem() || isa<CallInst>(I)))
- return InstructionsState::invalid();
- unsigned InstOpcode = I->getOpcode();
- if (IsBinOp && isa<BinaryOperator>(I)) {
- if (BinOpHelper.add(I))
- continue;
- } else if (IsCastOp && isa<CastInst>(I)) {
- Value *Op0 = MainOp->getOperand(0);
- Type *Ty0 = Op0->getType();
- Value *Op1 = I->getOperand(0);
- Type *Ty1 = Op1->getType();
- if (Ty0 == Ty1) {
- if (InstOpcode == Opcode || InstOpcode == AltOpcode)
- continue;
- if (Opcode == AltOpcode) {
- assert(isValidForAlternation(Opcode) &&
- isValidForAlternation(InstOpcode) &&
- "Cast isn't safe for alternation, logic needs to be updated!");
- AltOpcode = InstOpcode;
- AltOp = I;
- continue;
- }
- }
- } else if (auto *Inst = dyn_cast<CmpInst>(I); Inst && IsCmpOp) {
- auto *BaseInst = cast<CmpInst>(MainOp);
- Type *Ty0 = BaseInst->getOperand(0)->getType();
- Type *Ty1 = Inst->getOperand(0)->getType();
- if (Ty0 == Ty1) {
- assert(InstOpcode == Opcode && "Expected same CmpInst opcode.");
- assert(InstOpcode == AltOpcode &&
- "Alternate instructions are only supported by BinaryOperator "
- "and CastInst.");
- // Check for compatible operands. If the corresponding operands are not
- // compatible - need to perform alternate vectorization.
- CmpInst::Predicate CurrentPred = Inst->getPredicate();
- CmpInst::Predicate SwappedCurrentPred =
- CmpInst::getSwappedPredicate(CurrentPred);
-
- if ((VL.size() == 2 || SwappedPredsCompatible) &&
- (BasePred == CurrentPred || BasePred == SwappedCurrentPred))
- continue;
-
- if (isCmpSameOrSwapped(BaseInst, Inst, TLI))
- continue;
- auto *AltInst = cast<CmpInst>(AltOp);
- if (MainOp != AltOp) {
- if (isCmpSameOrSwapped(AltInst, Inst, TLI))
- continue;
- } else if (BasePred != CurrentPred) {
- assert(
- isValidForAlternation(InstOpcode) &&
- "CmpInst isn't safe for alternation, logic needs to be updated!");
- AltOp = I;
- continue;
- }
- CmpInst::Predicate AltPred = AltInst->getPredicate();
- if (BasePred == CurrentPred || BasePred == SwappedCurrentPred ||
- AltPred == CurrentPred || AltPred == SwappedCurrentPred)
- continue;
- }
- } else if (InstOpcode == Opcode) {
- assert(InstOpcode == AltOpcode &&
- "Alternate instructions are only supported by BinaryOperator and "
- "CastInst.");
- if (auto *Gep = dyn_cast<GetElementPtrInst>(I)) {
- if (Gep->getNumOperands() != 2 ||
- Gep->getOperand(0)->getType() != MainOp->getOperand(0)->getType())
- return InstructionsState::invalid();
- } else if (auto *EI = dyn_cast<ExtractElementInst>(I)) {
- if (!isVectorLikeInstWithConstOps(EI))
- return InstructionsState::invalid();
- } else if (auto *LI = dyn_cast<LoadInst>(I)) {
- auto *BaseLI = cast<LoadInst>(MainOp);
- if (!LI->isSimple() || !BaseLI->isSimple())
- return InstructionsState::invalid();
- } else if (auto *Call = dyn_cast<CallInst>(I)) {
- auto *CallBase = cast<CallInst>(MainOp);
- Intrinsic::ID ID = getVectorIntrinsicIDForCall(Call, &TLI);
- Intrinsic::ID Equivalent = isEquivalentIntrinsicID(ID, BaseID);
- if (Call->getCalledFunction() != CallBase->getCalledFunction() &&
- isEquivalentIntrinsicID(Equivalent, Intrinsic::fmuladd) ==
- Intrinsic::not_intrinsic)
- return InstructionsState::invalid();
- if (Call->hasOperandBundles() &&
- (!CallBase->hasOperandBundles() ||
- !std::equal(Call->op_begin() + Call->getBundleOperandsStartIndex(),
- Call->op_begin() + Call->getBundleOperandsEndIndex(),
- CallBase->op_begin() +
- CallBase->getBundleOperandsStartIndex())))
- return InstructionsState::invalid();
- if (ID != BaseID && Equivalent == Intrinsic::not_intrinsic)
- return InstructionsState::invalid();
- if (!ID) {
- SmallVector<VFInfo> Mappings = VFDatabase(*Call).getMappings(*Call);
- if (Mappings.size() != BaseMappings.size() ||
- Mappings.front().ISA != BaseMappings.front().ISA ||
- Mappings.front().ScalarName != BaseMappings.front().ScalarName ||
- Mappings.front().VectorName != BaseMappings.front().VectorName ||
- Mappings.front().Shape.VF != BaseMappings.front().Shape.VF ||
- Mappings.front().Shape.Parameters !=
- BaseMappings.front().Shape.Parameters)
- return InstructionsState::invalid();
- }
- }
- continue;
- }
- return InstructionsState::invalid();
- }
-
- if (IsBinOp) {
- if (!BinOpHelper.hasDefinedMainOpcode() ||
- !BinOpHelper.hasDefinedAltOpcode())
- return InstructionsState::invalid();
- MainOp = findInstructionWithOpcode(VL, BinOpHelper.getMainOpcode());
- assert(MainOp && "Cannot find MainOp with Opcode from BinOpHelper.");
- AltOp = findInstructionWithOpcode(VL, BinOpHelper.getAltOpcode());
- assert(AltOp && "Cannot find AltOp with Opcode from BinOpHelper.");
- } else if (auto *CB = dyn_cast<CallInst>(MainOp);
- CB &&
- getVectorIntrinsicIDForCall(CB, &TLI) == Intrinsic::fmuladd) {
- // fma and fmuladd share a single vector fma node; use the fma as the
- // representative so the fused form is not weakened to fmuladd.
- auto *It = find_if(VL, [&](Value *V) {
- auto *CI = dyn_cast<CallInst>(V);
- return CI && getVectorIntrinsicIDForCall(CI, &TLI) == Intrinsic::fma;
- });
- if (It != VL.end())
- MainOp = AltOp = cast<Instruction>(*It);
- }
- assert((MainOp == AltOp || !allSameOpcode(VL)) &&
- "Incorrect implementation of allSameOpcode.");
- InstructionsState S(MainOp, AltOp);
- assert(all_of(VL,
- [&](Value *V) {
- return isa<PoisonValue>(V) ||
- S.getMatchingMainOpOrAltOp(cast<Instruction>(V));
- }) &&
- "Invalid InstructionsState.");
- return S;
-}
-
/// Returns true if widened type of \p Ty elements with size \p Sz represents
/// full vector type, i.e. adding extra element results in extra parts upon type
/// legalization.
@@ -7507,12 +7219,6 @@ static bool areTwoInsertFromSameBuildVector(
return false;
}
-/// Checks if the specified instruction \p I is an alternate operation for
-/// the given \p MainOp and \p AltOp instructions.
-static bool isAlternateInstruction(Instruction *I, Instruction *MainOp,
- Instruction *AltOp,
- const TargetLibraryInfo &TLI);
-
std::optional<BoUpSLP::OrdersType>
BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom,
bool IgnoreReorder) {
@@ -13706,30 +13412,6 @@ static bool isMainInstruction(Instruction *I, Instruction *MainOp,
return InstructionsState(MainOp, AltOp).getMatchingMainOpOrAltOp(I) == MainOp;
}
-static bool isAlternateInstruction(Instruction *I, Instruction *MainOp,
- Instruction *AltOp,
- const TargetLibraryInfo &TLI) {
- if (auto *MainCI = dyn_cast<CmpInst>(MainOp)) {
- auto *AltCI = cast<CmpInst>(AltOp);
- CmpInst::Predicate MainP = MainCI->getPredicate();
- [[maybe_unused]] CmpInst::Predicate AltP = AltCI->getPredicate();
- assert(MainP != AltP && "Expected different main/alternate predicates.");
- auto *CI = cast<CmpInst>(I);
- if (isCmpSameOrSwapped(MainCI, CI, TLI))
- return false;
- if (isCmpSameOrSwapped(AltCI, CI, TLI))
- return true;
- CmpInst::Predicate P = CI->getPredicate();
- CmpInst::Predicate SwappedP = CmpInst::getSwappedPredicate(P);
-
- assert((MainP == P || AltP == P || MainP == SwappedP || AltP == SwappedP) &&
- "CmpInst expected to match either main or alternate predicate or "
- "their swap.");
- return MainP != P && MainP != SwappedP;
- }
- return InstructionsState(MainOp, AltOp).getMatchingMainOpOrAltOp(I) == AltOp;
-}
-
TTI::OperandValueInfo BoUpSLP::getOperandInfo(ArrayRef<Value *> Ops) const {
assert(!Ops.empty());
const auto *Op0 = Ops.front();
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
index 54cb526a10eab..85f34eca235e5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.cpp
@@ -10,18 +10,23 @@
#include "SLPUtils.h"
#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
+#include <algorithm>
#include <array>
#include <cassert>
#include <utility>
@@ -469,4 +474,304 @@ bool InstructionsState::isNonSchedulable(Value *V) const {
doesNotNeedToBeScheduled(V);
}
+/// Find an instruction with a specific opcode in VL.
+/// \param VL Array of values to search through. Must contain only Instructions
+/// and PoisonValues.
+/// \param Opcode The instruction opcode to search for
+/// \returns
+/// - The first instruction found with matching opcode
+/// - nullptr if no matching instruction is found
+static Instruction *findInstructionWithOpcode(ArrayRef<Value *> VL,
+ unsigned Opcode) {
+ for (Value *V : VL) {
+ if (isa<PoisonValue>(V))
+ continue;
+ assert(isa<Instruction>(V) && "Only accepts PoisonValue and Instruction.");
+ auto *Inst = cast<Instruction>(V);
+ if (Inst->getOpcode() == Opcode)
+ return Inst;
+ }
+ return nullptr;
+}
+
+/// Checks if the provided operands of 2 cmp instructions are compatible, i.e.
+/// compatible instructions or constants, or just some other regular values.
+static bool areCompatibleCmpOps(Value *BaseOp0, Value *BaseOp1, Value *Op0,
+ Value *Op1, const TargetLibraryInfo &TLI) {
+ return (isConstant(BaseOp0) && isConstant(Op0)) ||
+ (isConstant(BaseOp1) && isConstant(Op1)) ||
+ (!isa<Instruction>(BaseOp0) && !isa<Instruction>(Op0) &&
+ !isa<Instruction>(BaseOp1) && !isa<Instruction>(Op1)) ||
+ BaseOp0 == Op0 || BaseOp1 == Op1 ||
+ getSameOpcode({BaseOp0, Op0}, TLI) ||
+ getSameOpcode({BaseOp1, Op1}, TLI);
+}
+
+/// \returns true if a compare instruction \p CI has similar "look" and
+/// same predicate as \p BaseCI, "as is" or with its operands and predicate
+/// swapped, false otherwise.
+static bool isCmpSameOrSwapped(const CmpInst *BaseCI, const CmpInst *CI,
+ const TargetLibraryInfo &TLI) {
+ assert(BaseCI->getOperand(0)->getType() == CI->getOperand(0)->getType() &&
+ "Assessing comparisons of different types?");
+ CmpInst::Predicate BasePred = BaseCI->getPredicate();
+ CmpInst::Predicate Pred = CI->getPredicate();
+ CmpInst::Predicate SwappedPred = CmpInst::getSwappedPredicate(Pred);
+
+ Value *BaseOp0 = BaseCI->getOperand(0);
+ Value *BaseOp1 = BaseCI->getOperand(1);
+ Value *Op0 = CI->getOperand(0);
+ Value *Op1 = CI->getOperand(1);
+
+ return (BasePred == Pred &&
+ areCompatibleCmpOps(BaseOp0, BaseOp1, Op0, Op1, TLI)) ||
+ (BasePred == SwappedPred &&
+ areCompatibleCmpOps(BaseOp0, BaseOp1, Op1, Op0, TLI));
+}
+
+InstructionsState getSameOpcode(ArrayRef<Value *> VL,
+ const TargetLibraryInfo &TLI) {
+ // Make sure these are all Instructions.
+ if (!all_of(VL, IsaPred<Instruction, PoisonValue>))
+ return InstructionsState::invalid();
+
+ auto *It = find_if(VL, IsaPred<Instruction>);
+ if (It == VL.end())
+ return InstructionsState::invalid();
+
+ Instruction *MainOp = cast<Instruction>(*It);
+ unsigned InstCnt = std::count_if(It, VL.end(), IsaPred<Instruction>);
+ if ((VL.size() > 2 && !isa<PHINode>(MainOp) && InstCnt < VL.size() / 2) ||
+ (VL.size() == 2 && InstCnt < 2))
+ return InstructionsState::invalid();
+
+ bool IsCastOp = isa<CastInst>(MainOp);
+ bool IsBinOp = isa<BinaryOperator>(MainOp);
+ bool IsCmpOp = isa<CmpInst>(MainOp);
+ CmpInst::Predicate BasePred = IsCmpOp ? cast<CmpInst>(MainOp)->getPredicate()
+ : CmpInst::BAD_ICMP_PREDICATE;
+ Instruction *AltOp = MainOp;
+ unsigned Opcode = MainOp->getOpcode();
+ unsigned AltOpcode = Opcode;
+
+ BinOpSameOpcodeHelper BinOpHelper(MainOp);
+ bool SwappedPredsCompatible = IsCmpOp && [&]() {
+ SetVector<unsigned> UniquePreds, UniqueNonSwappedPreds;
+ UniquePreds.insert(BasePred);
+ UniqueNonSwappedPreds.insert(BasePred);
+ for (Value *V : VL) {
+ auto *I = dyn_cast<CmpInst>(V);
+ if (!I)
+ return false;
+ CmpInst::Predicate CurrentPred = I->getPredicate();
+ CmpInst::Predicate SwappedCurrentPred =
+ CmpInst::getSwappedPredicate(CurrentPred);
+ UniqueNonSwappedPreds.insert(CurrentPred);
+ if (!UniquePreds.contains(CurrentPred) &&
+ !UniquePreds.contains(SwappedCurrentPred))
+ UniquePreds.insert(CurrentPred);
+ }
+ // Total number of predicates > 2, but if consider swapped predicates
+ // compatible only 2, consider swappable predicates as compatible opcodes,
+ // not alternate.
+ return UniqueNonSwappedPreds.size() > 2 && UniquePreds.size() == 2;
+ }();
+ // Check for one alternate opcode from another BinaryOperator.
+ // TODO - generalize to support all operators (types, calls etc.).
+ Intrinsic::ID BaseID = 0;
+ SmallVector<VFInfo> BaseMappings;
+ if (auto *CallBase = dyn_cast<CallInst>(MainOp)) {
+ BaseID = getVectorIntrinsicIDForCall(CallBase, &TLI);
+ BaseMappings = VFDatabase(*CallBase).getMappings(*CallBase);
+ if (!isTriviallyVectorizable(BaseID) && BaseMappings.empty())
+ return InstructionsState::invalid();
+ }
+ bool AnyPoison = InstCnt != VL.size();
+ // Check MainOp too to be sure that it matches the requirements for the
+ // instructions.
+ for (Value *V : iterator_range(It, VL.end())) {
+ auto *I = dyn_cast<Instruction>(V);
+ if (!I)
+ continue;
+
+ // Cannot combine poison and divisions.
+ // TODO: do some smart analysis of the CallInsts to exclude divide-like
+ // intrinsics/functions only.
+ if (AnyPoison && (I->isIntDivRem() || I->isFPDivRem() || isa<CallInst>(I)))
+ return InstructionsState::invalid();
+ unsigned InstOpcode = I->getOpcode();
+ if (IsBinOp && isa<BinaryOperator>(I)) {
+ if (BinOpHelper.add(I))
+ continue;
+ } else if (IsCastOp && isa<CastInst>(I)) {
+ Value *Op0 = MainOp->getOperand(0);
+ Type *Ty0 = Op0->getType();
+ Value *Op1 = I->getOperand(0);
+ Type *Ty1 = Op1->getType();
+ if (Ty0 == Ty1) {
+ if (InstOpcode == Opcode || InstOpcode == AltOpcode)
+ continue;
+ if (Opcode == AltOpcode) {
+ assert(isValidForAlternation(Opcode) &&
+ isValidForAlternation(InstOpcode) &&
+ "Cast isn't safe for alternation, logic needs to be updated!");
+ AltOpcode = InstOpcode;
+ AltOp = I;
+ continue;
+ }
+ }
+ } else if (auto *Inst = dyn_cast<CmpInst>(I); Inst && IsCmpOp) {
+ auto *BaseInst = cast<CmpInst>(MainOp);
+ Type *Ty0 = BaseInst->getOperand(0)->getType();
+ Type *Ty1 = Inst->getOperand(0)->getType();
+ if (Ty0 == Ty1) {
+ assert(InstOpcode == Opcode && "Expected same CmpInst opcode.");
+ assert(InstOpcode == AltOpcode &&
+ "Alternate instructions are only supported by BinaryOperator "
+ "and CastInst.");
+ // Check for compatible operands. If the corresponding operands are not
+ // compatible - need to perform alternate vectorization.
+ CmpInst::Predicate CurrentPred = Inst->getPredicate();
+ CmpInst::Predicate SwappedCurrentPred =
+ CmpInst::getSwappedPredicate(CurrentPred);
+
+ if ((VL.size() == 2 || SwappedPredsCompatible) &&
+ (BasePred == CurrentPred || BasePred == SwappedCurrentPred))
+ continue;
+
+ if (isCmpSameOrSwapped(BaseInst, Inst, TLI))
+ continue;
+ auto *AltInst = cast<CmpInst>(AltOp);
+ if (MainOp != AltOp) {
+ if (isCmpSameOrSwapped(AltInst, Inst, TLI))
+ continue;
+ } else if (BasePred != CurrentPred) {
+ assert(
+ isValidForAlternation(InstOpcode) &&
+ "CmpInst isn't safe for alternation, logic needs to be updated!");
+ AltOp = I;
+ continue;
+ }
+ CmpInst::Predicate AltPred = AltInst->getPredicate();
+ if (BasePred == CurrentPred || BasePred == SwappedCurrentPred ||
+ AltPred == CurrentPred || AltPred == SwappedCurrentPred)
+ continue;
+ }
+ } else if (InstOpcode == Opcode) {
+ assert(InstOpcode == AltOpcode &&
+ "Alternate instructions are only supported by BinaryOperator and "
+ "CastInst.");
+ if (auto *Gep = dyn_cast<GetElementPtrInst>(I)) {
+ if (Gep->getNumOperands() != 2 ||
+ Gep->getOperand(0)->getType() != MainOp->getOperand(0)->getType())
+ return InstructionsState::invalid();
+ } else if (auto *EI = dyn_cast<ExtractElementInst>(I)) {
+ if (!isVectorLikeInstWithConstOps(EI))
+ return InstructionsState::invalid();
+ } else if (auto *LI = dyn_cast<LoadInst>(I)) {
+ auto *BaseLI = cast<LoadInst>(MainOp);
+ if (!LI->isSimple() || !BaseLI->isSimple())
+ return InstructionsState::invalid();
+ } else if (auto *Call = dyn_cast<CallInst>(I)) {
+ auto *CallBase = cast<CallInst>(MainOp);
+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(Call, &TLI);
+ Intrinsic::ID Equivalent = isEquivalentIntrinsicID(ID, BaseID);
+ if (Call->getCalledFunction() != CallBase->getCalledFunction() &&
+ isEquivalentIntrinsicID(Equivalent, Intrinsic::fmuladd) ==
+ Intrinsic::not_intrinsic)
+ return InstructionsState::invalid();
+ if (Call->hasOperandBundles() &&
+ (!CallBase->hasOperandBundles() ||
+ !std::equal(Call->op_begin() + Call->getBundleOperandsStartIndex(),
+ Call->op_begin() + Call->getBundleOperandsEndIndex(),
+ CallBase->op_begin() +
+ CallBase->getBundleOperandsStartIndex())))
+ return InstructionsState::invalid();
+ if (ID != BaseID && Equivalent == Intrinsic::not_intrinsic)
+ return InstructionsState::invalid();
+ if (!ID) {
+ SmallVector<VFInfo> Mappings = VFDatabase(*Call).getMappings(*Call);
+ if (Mappings.size() != BaseMappings.size() ||
+ Mappings.front().ISA != BaseMappings.front().ISA ||
+ Mappings.front().ScalarName != BaseMappings.front().ScalarName ||
+ Mappings.front().VectorName != BaseMappings.front().VectorName ||
+ Mappings.front().Shape.VF != BaseMappings.front().Shape.VF ||
+ Mappings.front().Shape.Parameters !=
+ BaseMappings.front().Shape.Parameters)
+ return InstructionsState::invalid();
+ }
+ }
+ continue;
+ }
+ return InstructionsState::invalid();
+ }
+
+ if (IsBinOp) {
+ if (!BinOpHelper.hasDefinedMainOpcode() ||
+ !BinOpHelper.hasDefinedAltOpcode())
+ return InstructionsState::invalid();
+ MainOp = findInstructionWithOpcode(VL, BinOpHelper.getMainOpcode());
+ assert(MainOp && "Cannot find MainOp with Opcode from BinOpHelper.");
+ AltOp = findInstructionWithOpcode(VL, BinOpHelper.getAltOpcode());
+ assert(AltOp && "Cannot find AltOp with Opcode from BinOpHelper.");
+ } else if (auto *CB = dyn_cast<CallInst>(MainOp);
+ CB &&
+ getVectorIntrinsicIDForCall(CB, &TLI) == Intrinsic::fmuladd) {
+ // fma and fmuladd share a single vector fma node; use the fma as the
+ // representative so the fused form is not weakened to fmuladd.
+ auto *It = find_if(VL, [&](Value *V) {
+ auto *CI = dyn_cast<CallInst>(V);
+ return CI && getVectorIntrinsicIDForCall(CI, &TLI) == Intrinsic::fma;
+ });
+ if (It != VL.end())
+ MainOp = AltOp = cast<Instruction>(*It);
+ }
+ assert((MainOp == AltOp || !allSameOpcode(VL)) &&
+ "Incorrect implementation of allSameOpcode.");
+ InstructionsState S(MainOp, AltOp);
+ assert(all_of(VL,
+ [&](Value *V) {
+ return isa<PoisonValue>(V) ||
+ S.getMatchingMainOpOrAltOp(cast<Instruction>(V));
+ }) &&
+ "Invalid InstructionsState.");
+ return S;
+}
+
+std::pair<Instruction *, SmallVector<Value *>>
+convertTo(Instruction *I, const InstructionsState &S) {
+ Instruction *SelectedOp = S.getMatchingMainOpOrAltOp(I);
+ assert(SelectedOp && "Cannot convert the instruction.");
+ if (I->isBinaryOp()) {
+ BinOpSameOpcodeHelper Converter(I);
+ return std::make_pair(SelectedOp, Converter.getOperand(SelectedOp));
+ }
+ // Use args() to skip the trailing callee operand in CallInst::operands().
+ if (auto *CI = dyn_cast<CallInst>(I))
+ return std::make_pair(SelectedOp, SmallVector<Value *>(CI->args()));
+ return std::make_pair(SelectedOp, SmallVector<Value *>(I->operands()));
+}
+
+bool isAlternateInstruction(Instruction *I, Instruction *MainOp,
+ Instruction *AltOp, const TargetLibraryInfo &TLI) {
+ if (auto *MainCI = dyn_cast<CmpInst>(MainOp)) {
+ auto *AltCI = cast<CmpInst>(AltOp);
+ CmpInst::Predicate MainP = MainCI->getPredicate();
+ [[maybe_unused]] CmpInst::Predicate AltP = AltCI->getPredicate();
+ assert(MainP != AltP && "Expected different main/alternate predicates.");
+ auto *CI = cast<CmpInst>(I);
+ if (isCmpSameOrSwapped(MainCI, CI, TLI))
+ return false;
+ if (isCmpSameOrSwapped(AltCI, CI, TLI))
+ return true;
+ CmpInst::Predicate P = CI->getPredicate();
+ CmpInst::Predicate SwappedP = CmpInst::getSwappedPredicate(P);
+
+ assert((MainP == P || AltP == P || MainP == SwappedP || AltP == SwappedP) &&
+ "CmpInst expected to match either main or alternate predicate or "
+ "their swap.");
+ return MainP != P && MainP != SwappedP;
+ }
+ return InstructionsState(MainOp, AltOp).getMatchingMainOpOrAltOp(I) == AltOp;
+}
} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
index 9f2bd8b5d6aca..7f11383dd0e25 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCompatibilityAnalysis.h
@@ -16,6 +16,7 @@
#ifndef LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPCOMPATIBILITYANALYSIS_H
#define LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPCOMPATIBILITYANALYSIS_H
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitmaskEnum.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
@@ -27,6 +28,7 @@
namespace llvm {
class Constant;
+class TargetLibraryInfo;
class Value;
} // namespace llvm
@@ -261,6 +263,19 @@ class InstructionsState {
}
};
+/// \returns analysis of the Instructions in \p VL described in
+/// InstructionsState, the Opcode that we suppose the whole list
+/// could be vectorized even if its structure is diverse.
+InstructionsState getSameOpcode(ArrayRef<Value *> VL,
+ const TargetLibraryInfo &TLI);
+
+std::pair<Instruction *, SmallVector<Value *>>
+convertTo(Instruction *I, const InstructionsState &S);
+
+/// Checks if the specified instruction \p I is an alternate operation for
+/// the given \p MainOp and \p AltOp instructions.
+bool isAlternateInstruction(Instruction *I, Instruction *MainOp,
+ Instruction *AltOp, const TargetLibraryInfo &TLI);
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPCOMPATIBILITYANALYSIS_H
More information about the llvm-commits
mailing list