[llvm] [DAG] isKnownNeverNaN - add DemandedElts element mask to isKnownNeverNaN calls (PR #135952)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 16 04:32:19 PDT 2025
https://github.com/RKSimon created https://github.com/llvm/llvm-project/pull/135952
Matches what we've done for computeKnownBits etc. to improve vector handling
>From 7648629831efd3133aadd6ee485a16b9fd9239d2 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 16 Apr 2025 12:29:53 +0100
Subject: [PATCH] [DAG] isKnownNeverNaN - add DemandedElts element mask to
isKnownNeverZero calls.
Matches what we've done for computeKnownBits etc. to improve vector handling
---
llvm/include/llvm/CodeGen/SelectionDAG.h | 14 ++++
llvm/include/llvm/CodeGen/TargetLowering.h | 1 +
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 72 +++++++++++++------
.../CodeGen/SelectionDAG/TargetLowering.cpp | 1 +
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 7 +-
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h | 5 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 5 +-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 5 +-
8 files changed, 75 insertions(+), 35 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 63423463eeee2..2ab6b4d3027e9 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -2142,11 +2142,25 @@ class SelectionDAG {
/// X|Cst == X+Cst iff X&Cst = 0.
bool isBaseWithConstantOffset(SDValue Op) const;
+ /// Test whether the given SDValue (or all elements of it, if it is a
+ /// vector) is known to never be NaN in \p DemandedElts. If \p SNaN is true,
+ /// returns if \p Op is known to never be a signaling NaN (it may still be a
+ /// qNaN).
+ bool isKnownNeverNaN(SDValue Op, const APInt &DemandedElts, bool SNaN = false,
+ unsigned Depth = 0) const;
+
/// Test whether the given SDValue (or all elements of it, if it is a
/// vector) is known to never be NaN. If \p SNaN is true, returns if \p Op is
/// known to never be a signaling NaN (it may still be a qNaN).
bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const;
+ /// \returns true if \p Op is known to never be a signaling NaN in \p
+ /// DemandedElts.
+ bool isKnownNeverSNaN(SDValue Op, const APInt &DemandedElts,
+ unsigned Depth = 0) const {
+ return isKnownNeverNaN(Op, DemandedElts, true, Depth);
+ }
+
/// \returns true if \p Op is known to never be a signaling NaN.
bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const {
return isKnownNeverNaN(Op, true, Depth);
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 0a36975f4f625..00c36266a069f 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4283,6 +4283,7 @@ class TargetLowering : public TargetLoweringBase {
/// NaN. If \p sNaN is true, returns if \p Op is known to never be a signaling
/// NaN.
virtual bool isKnownNeverNaNForTargetNode(SDValue Op,
+ const APInt &DemandedElts,
const SelectionDAG &DAG,
bool SNaN = false,
unsigned Depth = 0) const;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 46fc8856640de..e519e3e60893f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -5599,7 +5599,22 @@ bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
(Op.getOpcode() == ISD::ADD || isADDLike(Op));
}
-bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
+bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN,
+ unsigned Depth) const {
+ EVT VT = Op.getValueType();
+
+ // Since the number of lanes in a scalable vector is unknown at compile time,
+ // we track one bit which is implicitly broadcast to all lanes. This means
+ // that all lanes in a scalable vector are considered demanded.
+ APInt DemandedElts = VT.isFixedLengthVector()
+ ? APInt::getAllOnes(VT.getVectorNumElements())
+ : APInt(1, 1);
+
+ return isKnownNeverNaN(Op, DemandedElts, SNaN, Depth);
+}
+
+bool SelectionDAG::isKnownNeverNaN(SDValue Op, const APInt &DemandedElts,
+ bool SNaN, unsigned Depth) const {
// If we're told that NaNs won't happen, assume they won't.
if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
return true;
@@ -5613,6 +5628,9 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
(SNaN && !C->getValueAPF().isSignaling());
}
+ if (!DemandedElts)
+ return false; // No demanded elts, better to assume we don't know anything.
+
unsigned Opcode = Op.getOpcode();
switch (Opcode) {
case ISD::FADD:
@@ -5655,21 +5673,21 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
case ISD::FLDEXP: {
if (SNaN)
return true;
- return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
+ return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
}
case ISD::FABS:
case ISD::FNEG:
case ISD::FCOPYSIGN: {
- return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
+ return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
}
case ISD::SELECT:
- return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
- isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
+ return isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1) &&
+ isKnownNeverNaN(Op.getOperand(2), DemandedElts, SNaN, Depth + 1);
case ISD::FP_EXTEND:
case ISD::FP_ROUND: {
if (SNaN)
return true;
- return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
+ return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1);
}
case ISD::SINT_TO_FP:
case ISD::UINT_TO_FP:
@@ -5691,8 +5709,8 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
case ISD::FMAXIMUMNUM: {
// Only one needs to be known not-nan, since it will be returned if the
// other ends up being one.
- return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
- isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
+ return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) ||
+ isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);
}
case ISD::FMINNUM_IEEE:
case ISD::FMAXNUM_IEEE: {
@@ -5700,33 +5718,41 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
return true;
// This can return a NaN if either operand is an sNaN, or if both operands
// are NaN.
- return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
- isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
- (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
- isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
+ return (isKnownNeverNaN(Op.getOperand(0), DemandedElts, false, Depth + 1) &&
+ isKnownNeverSNaN(Op.getOperand(1), DemandedElts, Depth + 1)) ||
+ (isKnownNeverNaN(Op.getOperand(1), DemandedElts, false, Depth + 1) &&
+ isKnownNeverSNaN(Op.getOperand(0), DemandedElts, Depth + 1));
}
case ISD::FMINIMUM:
case ISD::FMAXIMUM: {
// TODO: Does this quiet or return the origina NaN as-is?
- return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
- isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
+ return isKnownNeverNaN(Op.getOperand(0), DemandedElts, SNaN, Depth + 1) &&
+ isKnownNeverNaN(Op.getOperand(1), DemandedElts, SNaN, Depth + 1);
}
- case ISD::EXTRACT_VECTOR_ELT:
- case ISD::EXTRACT_SUBVECTOR: {
+ case ISD::EXTRACT_VECTOR_ELT: {
+ // TODO: Set DemandedSrcElts for constant index.
return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
}
+ case ISD::EXTRACT_SUBVECTOR: {
+ SDValue Src = Op.getOperand(0);
+ unsigned Idx = Op.getConstantOperandVal(1);
+ unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
+ APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
+ return isKnownNeverNaN(Src, DemandedSrcElts, SNaN, Depth + 1);
+ }
case ISD::BUILD_VECTOR: {
- for (const SDValue &Opnd : Op->ops())
- if (!isKnownNeverNaN(Opnd, SNaN, Depth + 1))
+ unsigned NumElts = Op.getNumOperands();
+ for (unsigned I = 0; I != NumElts; ++I)
+ if (DemandedElts[I] &&
+ !isKnownNeverNaN(Op.getOperand(I), SNaN, Depth + 1))
return false;
return true;
}
default:
- if (Opcode >= ISD::BUILTIN_OP_END ||
- Opcode == ISD::INTRINSIC_WO_CHAIN ||
- Opcode == ISD::INTRINSIC_W_CHAIN ||
- Opcode == ISD::INTRINSIC_VOID) {
- return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
+ if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
+ Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID) {
+ return TLI->isKnownNeverNaNForTargetNode(Op, DemandedElts, *this, SNaN,
+ Depth);
}
return false;
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 5308593b5c988..3995216e3d689 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -3928,6 +3928,7 @@ bool TargetLowering::canCreateUndefOrPoisonForTargetNode(
}
bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
+ const APInt &DemandedElts,
const SelectionDAG &DAG,
bool SNaN,
unsigned Depth) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 533ad349f7500..2846405a2538c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -5971,10 +5971,9 @@ unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
}
}
-bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
- const SelectionDAG &DAG,
- bool SNaN,
- unsigned Depth) const {
+bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
+ SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
+ unsigned Depth) const {
unsigned Opcode = Op.getOpcode();
switch (Opcode) {
case AMDGPUISD::FMIN_LEGACY:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 6705f86e15fc2..fa9d61ec37c24 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -321,9 +321,8 @@ class AMDGPUTargetLowering : public TargetLowering {
const MachineRegisterInfo &MRI,
unsigned Depth = 0) const override;
- bool isKnownNeverNaNForTargetNode(SDValue Op,
- const SelectionDAG &DAG,
- bool SNaN = false,
+ bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts,
+ const SelectionDAG &DAG, bool SNaN = false,
unsigned Depth = 0) const override;
bool isReassocProfitable(MachineRegisterInfo &MRI, Register N0,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index bd95bcd89e183..9181d03f9f593 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16687,6 +16687,7 @@ bool SITargetLowering::denormalsEnabledForType(
}
bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
+ const APInt &DemandedElts,
const SelectionDAG &DAG,
bool SNaN,
unsigned Depth) const {
@@ -16699,8 +16700,8 @@ bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
}
- return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DAG, SNaN,
- Depth);
+ return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DemandedElts,
+ DAG, SNaN, Depth);
}
// On older subtargets, global FP atomic instructions have a hardcoded FP mode
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index dc0634331caf9..c42366a1c04c8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -546,9 +546,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool isProfitableToHoist(Instruction *I) const override;
- bool isKnownNeverNaNForTargetNode(SDValue Op,
- const SelectionDAG &DAG,
- bool SNaN = false,
+ bool isKnownNeverNaNForTargetNode(SDValue Op, const APInt &DemandedElts,
+ const SelectionDAG &DAG, bool SNaN = false,
unsigned Depth = 0) const override;
AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
More information about the llvm-commits
mailing list