[llvm] IVDesc: unify RecurKinds IAnyOf and FAnyOf (PR #118393)
Ramkumar Ramachandra via llvm-commits
llvm-commits at lists.llvm.org
Thu May 22 06:16:10 PDT 2025
https://github.com/artagnon updated https://github.com/llvm/llvm-project/pull/118393
>From c9b44e93bf537ad2bacf3f0e83f163b911c03665 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra at codasip.com>
Date: Mon, 2 Dec 2024 19:37:06 +0000
Subject: [PATCH] IVDescriptors: cut wasteful FAnyOf checking (NFC)
Checking RecurKind::FAnyOf in isRecurrenceDescriptor() is wasted work
when it already checks RecurKind::IAnyOf. Affect a minor adjustment to
the code to facilitate skipping the RecurKind::FAnyOf check, and strip
the check. The patch has the side-effect of rewriting some flaky code,
which would match an ICmp having the reduction phi as an operand with
IAnyOf, and due to NumCmpSelectPatternInst != 1 (the select redux is
also matched), it would have to rely on failing to match an FCmp with
FAnyOf, setting NumCmpSelectPatternInst = 1, and successfully
vectorizing an IAnyOf pattern with the incorrect debug output. There is
a test for this already in select-cmp.ll:
select_i32_from_icmp_same_inputs.
Co-authored-by: Mel Chen <mel.chen at sifive.com>
---
llvm/include/llvm/Analysis/IVDescriptors.h | 42 +++++++++----------
llvm/lib/Analysis/IVDescriptors.cpp | 29 +++++--------
.../AArch64/AArch64TargetTransformInfo.cpp | 3 +-
.../Target/RISCV/RISCVTargetTransformInfo.h | 3 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 9 ++--
5 files changed, 35 insertions(+), 51 deletions(-)
diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h
index 140edff13a67f..2ad18356e8360 100644
--- a/llvm/include/llvm/Analysis/IVDescriptors.h
+++ b/llvm/include/llvm/Analysis/IVDescriptors.h
@@ -31,29 +31,27 @@ class StoreInst;
/// These are the kinds of recurrences that we support.
enum class RecurKind {
- None, ///< Not a recurrence.
- Add, ///< Sum of integers.
- Mul, ///< Product of integers.
- Or, ///< Bitwise or logical OR of integers.
- And, ///< Bitwise or logical AND of integers.
- Xor, ///< Bitwise or logical XOR of integers.
- SMin, ///< Signed integer min implemented in terms of select(cmp()).
- SMax, ///< Signed integer max implemented in terms of select(cmp()).
- UMin, ///< Unsigned integer min implemented in terms of select(cmp()).
- UMax, ///< Unsigned integer max implemented in terms of select(cmp()).
- FAdd, ///< Sum of floats.
- FMul, ///< Product of floats.
- FMin, ///< FP min implemented in terms of select(cmp()).
- FMax, ///< FP max implemented in terms of select(cmp()).
- FMinimum, ///< FP min with llvm.minimum semantics
- FMaximum, ///< FP max with llvm.maximum semantics
+ None, ///< Not a recurrence.
+ Add, ///< Sum of integers.
+ Mul, ///< Product of integers.
+ Or, ///< Bitwise or logical OR of integers.
+ And, ///< Bitwise or logical AND of integers.
+ Xor, ///< Bitwise or logical XOR of integers.
+ SMin, ///< Signed integer min implemented in terms of select(cmp()).
+ SMax, ///< Signed integer max implemented in terms of select(cmp()).
+ UMin, ///< Unsigned integer min implemented in terms of select(cmp()).
+ UMax, ///< Unsigned integer max implemented in terms of select(cmp()).
+ FAdd, ///< Sum of floats.
+ FMul, ///< Product of floats.
+ FMin, ///< FP min implemented in terms of select(cmp()).
+ FMax, ///< FP max implemented in terms of select(cmp()).
+ FMinimum, ///< FP min with llvm.minimum semantics
+ FMaximum, ///< FP max with llvm.maximum semantics
FMinimumNum, ///< FP min with llvm.minimumnum semantics
FMaximumNum, ///< FP max with llvm.maximumnum semantics
- FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
- IAnyOf, ///< Any_of reduction with select(icmp(),x,y) where one of (x,y) is
- ///< loop invariant, and both x and y are integer type.
- FAnyOf, ///< Any_of reduction with select(fcmp(),x,y) where one of (x,y) is
- ///< loop invariant, and both x and y are integer type.
+ FMulAdd, ///< Sum of float products with llvm.fmuladd(a * b + sum).
+ AnyOf, ///< AnyOf reduction with select(cmp(),x,y) where one of (x,y) is
+ ///< loop invariant.
IFindLastIV, ///< FindLast reduction with select(icmp(),x,y) where one of
///< (x,y) is increasing loop induction, and both x and y are
///< integer type.
@@ -253,7 +251,7 @@ class RecurrenceDescriptor {
/// Returns true if the recurrence kind is of the form
/// select(cmp(),x,y) where one of (x,y) is loop invariant.
static bool isAnyOfRecurrenceKind(RecurKind Kind) {
- return Kind == RecurKind::IAnyOf || Kind == RecurKind::FAnyOf;
+ return Kind == RecurKind::AnyOf;
}
/// Returns true if the recurrence kind is of the form
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index a273338670164..276eecd5a2d52 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -49,8 +49,7 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
case RecurKind::SMin:
case RecurKind::UMax:
case RecurKind::UMin:
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
case RecurKind::IFindLastIV:
case RecurKind::FFindLastIV:
return true;
@@ -417,11 +416,11 @@ bool RecurrenceDescriptor::AddReductionVar(
if (IsAPhi && Cur != Phi && !areAllUsesIn(Cur, VisitedInsts))
return false;
- if ((isIntMinMaxRecurrenceKind(Kind) || Kind == RecurKind::IAnyOf) &&
- (isa<ICmpInst>(Cur) || isa<SelectInst>(Cur)))
+ if (isIntMinMaxRecurrenceKind(Kind) && (isa<ICmpInst>(Cur) || IsASelect))
++NumCmpSelectPatternInst;
- if ((isFPMinMaxRecurrenceKind(Kind) || Kind == RecurKind::FAnyOf) &&
- (isa<FCmpInst>(Cur) || isa<SelectInst>(Cur)))
+ if (isFPMinMaxRecurrenceKind(Kind) && (isa<FCmpInst>(Cur) || IsASelect))
+ ++NumCmpSelectPatternInst;
+ if (isAnyOfRecurrenceKind(Kind) && IsASelect)
++NumCmpSelectPatternInst;
// Check whether we found a reduction operator.
@@ -656,8 +655,7 @@ RecurrenceDescriptor::isAnyOfPattern(Loop *Loop, PHINode *OrigPhi,
if (!Loop->isLoopInvariant(NonPhi))
return InstDesc(false, I);
- return InstDesc(I, isa<ICmpInst>(I->getOperand(0)) ? RecurKind::IAnyOf
- : RecurKind::FAnyOf);
+ return InstDesc(I, RecurKind::AnyOf);
}
// We are looking for loops that do something like this:
@@ -987,10 +985,10 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found a UMIN reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RecurKind::IAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
+ if (AddReductionVar(Phi, RecurKind::AnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
- LLVM_DEBUG(dbgs() << "Found an integer conditional select reduction PHI."
- << *Phi << "\n");
+ LLVM_DEBUG(dbgs() << "Found a conditional select reduction PHI." << *Phi
+ << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::IFindLastIV, TheLoop, FMF, RedDes, DB, AC,
@@ -1022,12 +1020,6 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found a float MIN reduction PHI." << *Phi << "\n");
return true;
}
- if (AddReductionVar(Phi, RecurKind::FAnyOf, TheLoop, FMF, RedDes, DB, AC, DT,
- SE)) {
- LLVM_DEBUG(dbgs() << "Found a float conditional select reduction PHI."
- << " PHI." << *Phi << "\n");
- return true;
- }
if (AddReductionVar(Phi, RecurKind::FMulAdd, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found an FMulAdd reduction PHI." << *Phi << "\n");
@@ -1154,8 +1146,7 @@ unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
return Instruction::Add;
case RecurKind::Mul:
return Instruction::Mul;
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
case RecurKind::Or:
return Instruction::Or;
case RecurKind::And:
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 3f10da23b3494..68aec80f07e1d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5072,8 +5072,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
case RecurKind::FMin:
case RecurKind::FMax:
case RecurKind::FMulAdd:
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
return true;
default:
return false;
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 53529d077fd54..dd0d764463d07 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -372,11 +372,10 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
case RecurKind::SMax:
case RecurKind::UMin:
case RecurKind::UMax:
- case RecurKind::IAnyOf:
case RecurKind::FMin:
case RecurKind::FMax:
return true;
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
case RecurKind::FAdd:
case RecurKind::FMulAdd:
// We can't promote f16/bf16 fadd reductions and scalable vectors can't be
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index fcb9da637dd37..5046df67fdd30 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -23069,8 +23069,7 @@ class HorizontalReduction {
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
case RecurKind::IFindLastIV:
case RecurKind::FFindLastIV:
case RecurKind::FMaximumNum:
@@ -23205,8 +23204,7 @@ class HorizontalReduction {
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
case RecurKind::IFindLastIV:
case RecurKind::FFindLastIV:
case RecurKind::FMaximumNum:
@@ -23306,8 +23304,7 @@ class HorizontalReduction {
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
- case RecurKind::IAnyOf:
- case RecurKind::FAnyOf:
+ case RecurKind::AnyOf:
case RecurKind::IFindLastIV:
case RecurKind::FFindLastIV:
case RecurKind::FMaximumNum:
More information about the llvm-commits
mailing list