[llvm] IVDescriptors: cut wasteful FAnyOf checking (PR #118393)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Tue Dec 3 03:36:24 PST 2024
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/118393
>From feaf59e0bf590f9064d949ad9ca4651fae8c3b9b Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Mon, 2 Dec 2024 19:37:06 +0000
Subject: [PATCH 1/2] IVDescriptors: cut wasteful FAnyOf checking (NFC)
Checking RecurKind::FAnyOf in isRecurrenceDescriptor() is wasted work
when it already checks RecurKind::IAnyOf. Affect a minor adjustment to
the code to facilitate skipping the RecurKind::FAnyOf check, and strip
the check. The patch has the side-effect of rewriting some flaky code,
which would match an ICmp having the reduction phi as an operand with
IAnyOf, and due to NumCmpSelectPatternInst != 1 (the select redux is
also matched), it would have to rely on failing to match an FCmp with
FAnyOf, setting NumCmpSelectPatternInst = 1, and successfully
vectorizing an IAnyOf pattern with the incorrect debug output. There is
a test for this already in select-cmp.ll:
select_i32_from_icmp_same_inputs.
---
llvm/lib/Analysis/IVDescriptors.cpp | 21 ++++++---------------
1 file changed, 6 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 23e11bdbeab4c5..9678fcdcbd7c0e 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -415,12 +415,9 @@ bool RecurrenceDescriptor::AddReductionVar(
if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))
return false;
- if ((isIntMinMaxRecurrenceKind(Kind) || Kind == RecurKind::IAnyOf) &&
- (isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))
- ++NumCmpSelectPatternInst;
- if ((isFPMinMaxRecurrenceKind(Kind) || Kind == RecurKind::FAnyOf) &&
- (isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
- ++NumCmpSelectPatternInst;
+ if (isIntMinMaxRecurrenceKind(Kind) || isAnyOfRecurrenceKind(Kind))
+ if (match(Cur, m_Select(m_Cmp(), m_Value(), m_Value())))
+ ++NumCmpSelectPatternInst;
// Check whether we found a reduction operator.
FoundReduxOp |= !IsAPhi && Cur != Start;
@@ -500,7 +497,7 @@ bool RecurrenceDescriptor::AddReductionVar(
// This means we have seen one but not the other instruction of the
// pattern or more than just a select and cmp. Zero implies that we saw a
// llvm.min/max intrinsic, which is always OK.
- if (isMinMaxRecurrenceKind(Kind) && NumCmpSelectPatternInst != 2 &&
+ if (isMinMaxRecurrenceKind(Kind) && NumCmpSelectPatternInst != 1 &&
NumCmpSelectPatternInst != 0)
return false;
@@ -889,8 +886,8 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
}
if (AddReductionVar(Phi, RecurKind::IAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
- LLVM_DEBUG(dbgs() << "Found an integer conditional select reduction PHI."
- << *Phi << "\n");
+ LLVM_DEBUG(dbgs() << "Found an conditional select reduction PHI." << *Phi
+ << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::FMul, TheLoop, FMF, RedDes, DB, AC, DT,
@@ -913,12 +910,6 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RecurKind::FAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
- SE)) {
- LLVM_DEBUG(dbgs() << "Found a float conditional select reduction PHI."
- << " PHI." << *Phi << "\n");
- return true;
- }
if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found an FMulAdd reduction PHI." << *Phi << "\n");
>From b70d6e3990a4580e90f80181d35a7772d639e57b Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Tue, 3 Dec 2024 11:06:18 +0000
Subject: [PATCH 2/2] IVDescriptors: strip FAnyOf altogether
---
llvm/include/llvm/Analysis/IVDescriptors.h | 14 +++++++-------
llvm/lib/Analysis/IVDescriptors.cpp | 16 +++++++---------
.../AArch64/AArch64TargetTransformInfo.cpp | 3 +--
llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h | 3 +--
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 10 ++++------
llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 11 +++++------
6 files changed, 25 insertions(+), 32 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 5d992faf99d27f..7e8fa85093ff24 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -48,10 +48,8 @@ enum class RecurKind {
FMinimum, ///< FP min with llvm.minimum semantics
FMaximum, ///< FP max with llvm.maximum semantics
FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
- IAnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
- ///< loop invariant, and both x and y are integer type.
- FAnyOf ///< Any_of reduction with select(fcmp(),x,y) where one of (x,y) is
- ///< loop invariant, and both x and y are integer type.
+ AnyOf, ///< Any_of reduction with select(cmp(),x,y) where one of (x,y) is
+ ///< loop invariant.
// TODO: Any_of reduction need not be restricted to integer type only.
};
@@ -156,7 +154,7 @@ class RecurrenceDescriptor {
static InstDesc isConditionalRdxPattern(RecurKind Kind, Instruction *I);
/// Returns the opcode corresponding to the RecurrenceKind.
- static unsigned getOpcode(RecurKind Kind);
+ static unsigned getOpcode(RecurKind Kind, Type *Ty);
/// Returns true if Phi is a reduction of type Kind and adds it to the
/// RecurrenceDescriptor. If either \p DB is non-null or \p AC and \p DT are
@@ -192,7 +190,9 @@ class RecurrenceDescriptor {
RecurKind getRecurrenceKind() const { return Kind; }
- unsigned getOpcode() const { return getOpcode(getRecurrenceKind()); }
+ unsigned getOpcode() const {
+ return getOpcode(getRecurrenceKind(), getRecurrenceType());
+ }
FastMathFlags getFastMathFlags() const { return FMF; }
@@ -233,7 +233,7 @@ class RecurrenceDescriptor {
/// Returns true if the recurrence kind is of the form
/// select(cmp(),x,y) where one of (x,y) is loop invariant.
static bool isAnyOfRecurrenceKind(RecurKind Kind) {
- return Kind == RecurKind::IAnyOf || Kind == RecurKind::FAnyOf;
+ return Kind == RecurKind::AnyOf;
}
/// Returns the type of the recurrence. This type can be narrower than the
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index 9678fcdcbd7c0e..f1a4690d443fc0 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -49,8 +49,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
case RecurKind::SMin:
case RecurKind::UMax:
case RecurKind::UMin:
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
return true;
}
return false;
@@ -651,8 +650,7 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
if (!Loop->isLoopInvariant(NonPhi))
return InstDesc(false, I);
- return InstDesc(I, isa<ICmpInst>(I->getOperand(0)) ? RecurKind::IAnyOf
- : RecurKind::FAnyOf);
+ return InstDesc(I, RecurKind::AnyOf);
}
RecurrenceDescriptor::InstDesc
@@ -884,7 +882,7 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RecurKind::IAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
+ if (AddReductionVar(Phi, RecurKind::AnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found an conditional select reduction PHI." << *Phi
<< "\n");
@@ -1017,7 +1015,7 @@ bool RecurrenceDescriptor::isFixedOrderRecurrence(PHINode *Phi, Loop *TheLoop,
return true;
}
-unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
+unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind, Type *Ty) {
switch (Kind) {
case RecurKind::Add:
return Instruction::Add;
@@ -1038,14 +1036,14 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
case RecurKind::SMin:
case RecurKind::UMax:
case RecurKind::UMin:
- case RecurKind::IAnyOf:
return Instruction::ICmp;
case RecurKind::FMax:
case RecurKind::FMin:
case RecurKind::FMaximum:
case RecurKind::FMinimum:
- case RecurKind::FAnyOf:
return Instruction::FCmp;
+ case RecurKind::AnyOf:
+ return Ty->isIntegerTy() ? Instruction::ICmp : Instruction::FCmp;
default:
llvm_unreachable("Unknown recurrence operation");
}
@@ -1054,7 +1052,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
SmallVector<Instruction *, 4>
RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const {
SmallVector<Instruction *, 4> ReductionOperations;
- unsigned RedOp = getOpcode(Kind);
+ unsigned RedOp = getOpcode();
// Search down from the Phi to the LoopExitInstr, looking for instructions
// with a single user of the correct type for the reduction.
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 5b333d33cffd52..aa3af71672eae3 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -4180,8 +4180,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
case RecurKind::FMin:
case RecurKind::FMax:
case RecurKind::FMulAdd:
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
return true;
default:
return false;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index bd90bfed6e2c95..e9dbd32d998f2d 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -348,8 +348,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
case RecurKind::FMin:
case RecurKind::FMax:
case RecurKind::FMulAdd:
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
return true;
default:
return false;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 33657c26356d65..7e42575b725869 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19241,7 +19241,7 @@ class HorizontalReduction {
/// Creates reduction operation with the current opcode.
static Value *createOp(IRBuilderBase &Builder, RecurKind Kind, Value *LHS,
Value *RHS, const Twine &Name, bool UseSelect) {
- unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind);
+ unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(Kind, LHS->getType());
switch (Kind) {
case RecurKind::Or:
if (UseSelect &&
@@ -20384,7 +20384,7 @@ class HorizontalReduction {
case RecurKind::Xor:
case RecurKind::FAdd:
case RecurKind::FMul: {
- unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
+ unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind, ScalarTy);
if (!AllConsts) {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
@@ -20513,8 +20513,7 @@ class HorizontalReduction {
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for repeated scalar.");
}
@@ -20610,8 +20609,7 @@ class HorizontalReduction {
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
case RecurKind::None:
llvm_unreachable("Unexpected reduction kind for reused scalars.");
}
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 8a44b5b176c46d..8903adbb738c56 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -550,7 +550,7 @@ Value *VPInstruction::generate(VPTransformState &State) {
}
// Reduce all of the unrolled parts into a single vector.
Value *ReducedPartRdx = RdxParts[0];
- unsigned Op = RecurrenceDescriptor::getOpcode(RK);
+ unsigned Op = RdxDesc.getOpcode();
if (RecurrenceDescriptor::isAnyOfRecurrenceKind(RK))
Op = Instruction::Or;
@@ -2130,8 +2130,7 @@ void VPReductionRecipe::execute(VPTransformState &State) {
createOrderedReduction(State.Builder, RdxDesc, NewVecOp, PrevInChain);
else
NewRed = State.Builder.CreateBinOp(
- (Instruction::BinaryOps)RdxDesc.getOpcode(Kind), PrevInChain,
- NewVecOp);
+ (Instruction::BinaryOps)RdxDesc.getOpcode(), PrevInChain, NewVecOp);
PrevInChain = NewRed;
NextInChain = NewRed;
} else {
@@ -2142,7 +2141,7 @@ void VPReductionRecipe::execute(VPTransformState &State) {
NewRed, PrevInChain);
else
NextInChain = State.Builder.CreateBinOp(
- (Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, PrevInChain);
+ (Instruction::BinaryOps)RdxDesc.getOpcode(), NewRed, PrevInChain);
}
State.set(this, NextInChain, /*IsScalar*/ true);
}
@@ -2179,8 +2178,8 @@ void VPReductionEVLRecipe::execute(VPTransformState &State) {
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind))
NewRed = createMinMaxOp(Builder, Kind, NewRed, Prev);
else
- NewRed = Builder.CreateBinOp(
- (Instruction::BinaryOps)RdxDesc.getOpcode(Kind), NewRed, Prev);
+ NewRed = Builder.CreateBinOp((Instruction::BinaryOps)RdxDesc.getOpcode(),
+ NewRed, Prev);
}
State.set(this, NewRed, /*IsScalar*/ true);
}
More information about the llvm-commits
mailing list