[llvm] [IR] Store fast-math flags in subclasses of Instruction (PR #191190)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 00:19:42 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-ir
Author: paperchalice
<details>
<summary>Changes</summary>
Move fast-math flags out from `Value`, because we are out of space of `Value::SubclassOptionalData` and it is incompatible with other optimization flags like `nneg`.
FP variant for `call/select/phi` is not introduced, because of `mutateType`, it may change the type of the `Instruction` instance, which may cause UB.
---
Patch is 26.75 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/191190.diff
9 Files Affected:
- (modified) llvm/include/llvm/IR/InstVisitor.h (+1)
- (modified) llvm/include/llvm/IR/InstrTypes.h (+74-9)
- (modified) llvm/include/llvm/IR/Instruction.def (+5-5)
- (modified) llvm/include/llvm/IR/Instruction.h (+2-1)
- (modified) llvm/include/llvm/IR/Instructions.h (+8-6)
- (modified) llvm/include/llvm/IR/Operator.h (+52-38)
- (modified) llvm/lib/IR/Instruction.cpp (+7-4)
- (modified) llvm/lib/IR/Instructions.cpp (+51-10)
- (modified) llvm/lib/IR/Operator.cpp (+48)
``````````diff
diff --git a/llvm/include/llvm/IR/InstVisitor.h b/llvm/include/llvm/IR/InstVisitor.h
index 4d5dd00b744b9..ff2b999b271a0 100644
--- a/llvm/include/llvm/IR/InstVisitor.h
+++ b/llvm/include/llvm/IR/InstVisitor.h
@@ -264,6 +264,7 @@ class InstVisitor {
RetTy visitCastInst(CastInst &I) { DELEGATE(UnaryInstruction);}
RetTy visitUnaryOperator(UnaryOperator &I) { DELEGATE(UnaryInstruction);}
RetTy visitBinaryOperator(BinaryOperator &I) { DELEGATE(Instruction);}
+ RetTy visitFPBinaryOperator(FPBinaryOperator &I) { DELEGATE(BinaryOperator); }
RetTy visitCmpInst(CmpInst &I) { DELEGATE(Instruction);}
RetTy visitUnaryInstruction(UnaryInstruction &I){ DELEGATE(Instruction);}
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 61dc5ebef1b1d..c8fac25d0e032 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -51,6 +51,15 @@ namespace Intrinsic {
typedef unsigned ID;
}
+/// Provide fast-math flags storage, instructions that support fast-math flags
+/// should inherit from this class.
+class FastMathFlagsStorage {
+ friend class FPMathOperator;
+
+protected:
+ unsigned short FMFValue = 0;
+};
+
//===----------------------------------------------------------------------===//
// UnaryInstruction Class
//===----------------------------------------------------------------------===//
@@ -98,7 +107,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(UnaryInstruction, Value)
// UnaryOperator Class
//===----------------------------------------------------------------------===//
-class UnaryOperator : public UnaryInstruction {
+class UnaryOperator : public UnaryInstruction, public FastMathFlagsStorage {
void AssertOK();
protected:
@@ -203,16 +212,12 @@ class BinaryOperator : public Instruction {
/// statically know what type of instruction you're going to create. These
/// helpers just save some typing.
#define HANDLE_BINARY_INST(N, OPC, CLASS) \
- static BinaryOperator *Create##OPC(Value *V1, Value *V2, \
- const Twine &Name = "") { \
- return Create(Instruction::OPC, V1, V2, Name); \
- }
+ static inline BinaryOperator *Create##OPC(Value *V1, Value *V2, \
+ const Twine &Name = "");
#include "llvm/IR/Instruction.def"
#define HANDLE_BINARY_INST(N, OPC, CLASS) \
- static BinaryOperator *Create##OPC(Value *V1, Value *V2, const Twine &Name, \
- InsertPosition InsertBefore) { \
- return Create(Instruction::OPC, V1, V2, Name, InsertBefore); \
- }
+ static inline BinaryOperator *Create##OPC( \
+ Value *V1, Value *V2, const Twine &Name, InsertPosition InsertBefore);
#include "llvm/IR/Instruction.def"
static BinaryOperator *
@@ -435,6 +440,66 @@ BinaryOperator *BinaryOperator::CreateDisjoint(BinaryOps Opc, Value *V1,
return BO;
}
+/// Binary operators support fast-math flags, users should not use this
+/// class directly, BinaryOperator can automatic create instructions with
+/// correct type.
+class FPBinaryOperator : public BinaryOperator, public FastMathFlagsStorage {
+ friend class Instruction;
+ friend class BinaryOperator;
+ LLVM_ABI FPBinaryOperator *cloneImpl() const;
+ using BinaryOperator::BinaryOperator;
+
+ LLVM_ABI static FPBinaryOperator *
+ Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name = Twine(),
+ InsertPosition InsertBefore = nullptr);
+
+public:
+ static bool classof(const Instruction *I) {
+ switch (I->getOpcode()) {
+ case Instruction::FAdd:
+ case Instruction::FSub:
+ case Instruction::FMul:
+ case Instruction::FDiv:
+ case Instruction::FRem:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ static bool classof(const Value *V) {
+ return isa<Instruction>(V) && classof(cast<Instruction>(V));
+ }
+};
+
+#define HANDLE_BINARY_INST(N, OPC, CLASS) \
+ BinaryOperator *BinaryOperator::Create##OPC(Value *V1, Value *V2, \
+ const Twine &Name) { \
+ if constexpr (Instruction::OPC == Instruction::FAdd || \
+ Instruction::OPC == Instruction::FSub || \
+ Instruction::OPC == Instruction::FMul || \
+ Instruction::OPC == Instruction::FDiv || \
+ Instruction::OPC == Instruction::FRem) \
+ return FPBinaryOperator::Create(Instruction::OPC, V1, V2, Name); \
+ else \
+ return Create(Instruction::OPC, V1, V2, Name); \
+ }
+#include "llvm/IR/Instruction.def"
+#define HANDLE_BINARY_INST(N, OPC, CLASS) \
+ BinaryOperator *BinaryOperator::Create##OPC( \
+ Value *V1, Value *V2, const Twine &Name, InsertPosition InsertBefore) { \
+ if constexpr (Instruction::OPC == Instruction::FAdd || \
+ Instruction::OPC == Instruction::FSub || \
+ Instruction::OPC == Instruction::FMul || \
+ Instruction::OPC == Instruction::FDiv || \
+ Instruction::OPC == Instruction::FRem) \
+ return FPBinaryOperator::Create(Instruction::OPC, V1, V2, Name, \
+ InsertBefore); \
+ else \
+ return Create(Instruction::OPC, V1, V2, Name, InsertBefore); \
+ }
+#include "llvm/IR/Instruction.def"
+
//===----------------------------------------------------------------------===//
// CastInst Class
//===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/Instruction.def b/llvm/include/llvm/IR/Instruction.def
index 89d214868e16b..c1c008d634616 100644
--- a/llvm/include/llvm/IR/Instruction.def
+++ b/llvm/include/llvm/IR/Instruction.def
@@ -146,17 +146,17 @@ HANDLE_UNARY_INST(13, FNeg , UnaryOperator)
// Standard binary operators...
FIRST_BINARY_INST(14)
HANDLE_BINARY_INST(14, Add , BinaryOperator)
-HANDLE_BINARY_INST(15, FAdd , BinaryOperator)
+HANDLE_BINARY_INST(15, FAdd , FPBinaryOperator)
HANDLE_BINARY_INST(16, Sub , BinaryOperator)
-HANDLE_BINARY_INST(17, FSub , BinaryOperator)
+HANDLE_BINARY_INST(17, FSub , FPBinaryOperator)
HANDLE_BINARY_INST(18, Mul , BinaryOperator)
-HANDLE_BINARY_INST(19, FMul , BinaryOperator)
+HANDLE_BINARY_INST(19, FMul , FPBinaryOperator)
HANDLE_BINARY_INST(20, UDiv , BinaryOperator)
HANDLE_BINARY_INST(21, SDiv , BinaryOperator)
-HANDLE_BINARY_INST(22, FDiv , BinaryOperator)
+HANDLE_BINARY_INST(22, FDiv , FPBinaryOperator)
HANDLE_BINARY_INST(23, URem , BinaryOperator)
HANDLE_BINARY_INST(24, SRem , BinaryOperator)
-HANDLE_BINARY_INST(25, FRem , BinaryOperator)
+HANDLE_BINARY_INST(25, FRem , FPBinaryOperator)
// Logical operators (integer operands)
HANDLE_BINARY_INST(26, Shl , BinaryOperator) // Shift left (logical)
diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h
index 0b57ad4d0a379..379eca2fd43a6 100644
--- a/llvm/include/llvm/IR/Instruction.h
+++ b/llvm/include/llvm/IR/Instruction.h
@@ -712,7 +712,8 @@ class Instruction : public User,
/// Convenience method to copy supported exact, fast-math, and (optionally)
/// wrapping flags from V to this instruction.
- LLVM_ABI void copyIRFlags(const Value *V, bool IncludeWrapFlags = true);
+ LLVM_ABI void copyIRFlags(const Value *V, bool IncludeWrapFlags = true,
+ bool IncludeFastMathFlags = true);
/// Logical 'and' of any supported wrapping, exact, and fast-math flags of
/// V and this instruction.
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index f4fa59ada9d05..2fa49b6fc4e28 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -1418,7 +1418,7 @@ class ICmpInst: public CmpInst {
/// to the constructor. It only operates on floating point values or packed
/// vectors of floating point values. The operands must be identical types.
/// Represents a floating point comparison operator.
-class FCmpInst: public CmpInst {
+class FCmpInst : public CmpInst, public FastMathFlagsStorage {
void AssertOK() {
assert(isFPPredicate() && "Invalid FCmp predicate value");
assert(getOperand(0)->getType() == getOperand(1)->getType() &&
@@ -1456,6 +1456,8 @@ class FCmpInst: public CmpInst {
Instruction *FlagsSource = nullptr)
: CmpInst(makeCmpResultType(LHS->getType()), Instruction::FCmp, Pred, LHS,
RHS, NameStr, nullptr, FlagsSource) {
+ if (FlagsSource)
+ copyFastMathFlags(FlagsSource);
AssertOK();
}
@@ -1518,7 +1520,7 @@ class FCmpInst: public CmpInst {
/// field to indicate whether or not this is a tail call. The rest of the bits
/// hold the calling convention of the call.
///
-class CallInst : public CallBase {
+class CallInst : public CallBase, public FastMathFlagsStorage {
CallInst(const CallInst &CI, AllocInfo AllocInfo);
/// Construct a CallInst from a range of arguments
@@ -1697,7 +1699,7 @@ CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
/// This class represents the LLVM 'select' instruction.
///
-class SelectInst : public Instruction {
+class SelectInst : public Instruction, public FastMathFlagsStorage {
constexpr static IntrusiveOperandsAllocMarker AllocMarker{3};
SelectInst(Value *C, Value *S1, Value *S2, const Twine &NameStr,
@@ -2648,7 +2650,7 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(InsertValueInst, Value)
// node, that can not exist in nature, but can be synthesized in a computer
// scientist's overactive imagination.
//
-class PHINode : public Instruction {
+class PHINode : public Instruction, public FastMathFlagsStorage {
constexpr static HungOffOperandsAllocMarker AllocMarker{};
/// The number of operands actually allocated. NumOperands is
@@ -4878,7 +4880,7 @@ class SExtInst : public CastInst {
//===----------------------------------------------------------------------===//
/// This class represents a truncation of floating point types.
-class FPTruncInst : public CastInst {
+class FPTruncInst : public CastInst, public FastMathFlagsStorage {
protected:
// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;
@@ -4909,7 +4911,7 @@ class FPTruncInst : public CastInst {
//===----------------------------------------------------------------------===//
/// This class represents an extension of floating point types.
-class FPExtInst : public CastInst {
+class FPExtInst : public CastInst, public FastMathFlagsStorage {
protected:
// Note: Instruction needs to be a friend here to call cloneImpl.
friend class Instruction;
diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h
index 2e0e71cbbf315..89f6b1f5ea1c6 100644
--- a/llvm/include/llvm/IR/Operator.h
+++ b/llvm/include/llvm/IR/Operator.h
@@ -201,6 +201,9 @@ class FPMathOperator : public Operator {
private:
friend class Instruction;
+ unsigned short &getFMFValue();
+ unsigned short getFMFValue() const;
+
/// 'Fast' means all bits are set.
void setFast(bool B) {
setHasAllowReassoc(B);
@@ -213,57 +216,59 @@ class FPMathOperator : public Operator {
}
void setHasAllowReassoc(bool B) {
- SubclassOptionalData =
- (SubclassOptionalData & ~FastMathFlags::AllowReassoc) |
- (B * FastMathFlags::AllowReassoc);
+ unsigned short &FMFValue = getFMFValue();
+ FMFValue = (FMFValue & ~FastMathFlags::AllowReassoc) |
+ (B * FastMathFlags::AllowReassoc);
}
void setHasNoNaNs(bool B) {
- SubclassOptionalData =
- (SubclassOptionalData & ~FastMathFlags::NoNaNs) |
- (B * FastMathFlags::NoNaNs);
+ unsigned short &FMFValue = getFMFValue();
+ FMFValue =
+ (FMFValue & ~FastMathFlags::NoNaNs) | (B * FastMathFlags::NoNaNs);
}
void setHasNoInfs(bool B) {
- SubclassOptionalData =
- (SubclassOptionalData & ~FastMathFlags::NoInfs) |
- (B * FastMathFlags::NoInfs);
+ unsigned short &FMFValue = getFMFValue();
+ FMFValue =
+ (FMFValue & ~FastMathFlags::NoInfs) | (B * FastMathFlags::NoInfs);
}
void setHasNoSignedZeros(bool B) {
- SubclassOptionalData =
- (SubclassOptionalData & ~FastMathFlags::NoSignedZeros) |
- (B * FastMathFlags::NoSignedZeros);
+ unsigned short &FMFValue = getFMFValue();
+ FMFValue = (FMFValue & ~FastMathFlags::NoSignedZeros) |
+ (B * FastMathFlags::NoSignedZeros);
}
void setHasAllowReciprocal(bool B) {
- SubclassOptionalData =
- (SubclassOptionalData & ~FastMathFlags::AllowReciprocal) |
- (B * FastMathFlags::AllowReciprocal);
+ unsigned short &FMFValue = getFMFValue();
+ FMFValue = (FMFValue & ~FastMathFlags::AllowReciprocal) |
+ (B * FastMathFlags::AllowReciprocal);
}
void setHasAllowContract(bool B) {
- SubclassOptionalData =
- (SubclassOptionalData & ~FastMathFlags::AllowContract) |
- (B * FastMathFlags::AllowContract);
+ unsigned short &FMFValue = getFMFValue();
+ FMFValue = (FMFValue & ~FastMathFlags::AllowContract) |
+ (B * FastMathFlags::AllowContract);
}
void setHasApproxFunc(bool B) {
- SubclassOptionalData =
- (SubclassOptionalData & ~FastMathFlags::ApproxFunc) |
- (B * FastMathFlags::ApproxFunc);
+ unsigned short &FMFValue = getFMFValue();
+ FMFValue = (FMFValue & ~FastMathFlags::ApproxFunc) |
+ (B * FastMathFlags::ApproxFunc);
}
/// Convenience function for setting multiple fast-math flags.
/// FMF is a mask of the bits to set.
void setFastMathFlags(FastMathFlags FMF) {
- SubclassOptionalData |= FMF.Flags;
+ unsigned short &FMFValue = getFMFValue();
+ FMFValue |= FMF.Flags;
}
/// Convenience function for copying all fast-math flags.
/// All values in FMF are transferred to this operator.
void copyFastMathFlags(FastMathFlags FMF) {
- SubclassOptionalData = FMF.Flags;
+ unsigned short &FMFValue = getFMFValue();
+ FMFValue = FMF.Flags;
}
/// Returns true if `Ty` is composed of a single kind of float-poing type
@@ -284,54 +289,63 @@ class FPMathOperator : public Operator {
public:
/// Test if this operation allows all non-strict floating-point transforms.
bool isFast() const {
- return ((SubclassOptionalData & FastMathFlags::AllowReassoc) != 0 &&
- (SubclassOptionalData & FastMathFlags::NoNaNs) != 0 &&
- (SubclassOptionalData & FastMathFlags::NoInfs) != 0 &&
- (SubclassOptionalData & FastMathFlags::NoSignedZeros) != 0 &&
- (SubclassOptionalData & FastMathFlags::AllowReciprocal) != 0 &&
- (SubclassOptionalData & FastMathFlags::AllowContract) != 0 &&
- (SubclassOptionalData & FastMathFlags::ApproxFunc) != 0);
+ unsigned short FMFValue = getFMFValue();
+ return ((FMFValue & FastMathFlags::AllowReassoc) != 0 &&
+ (FMFValue & FastMathFlags::NoNaNs) != 0 &&
+ (FMFValue & FastMathFlags::NoInfs) != 0 &&
+ (FMFValue & FastMathFlags::NoSignedZeros) != 0 &&
+ (FMFValue & FastMathFlags::AllowReciprocal) != 0 &&
+ (FMFValue & FastMathFlags::AllowContract) != 0 &&
+ (FMFValue & FastMathFlags::ApproxFunc) != 0);
}
/// Test if this operation may be simplified with reassociative transforms.
bool hasAllowReassoc() const {
- return (SubclassOptionalData & FastMathFlags::AllowReassoc) != 0;
+ unsigned short FMFValue = getFMFValue();
+ return (FMFValue & FastMathFlags::AllowReassoc) != 0;
}
/// Test if this operation's arguments and results are assumed not-NaN.
bool hasNoNaNs() const {
- return (SubclassOptionalData & FastMathFlags::NoNaNs) != 0;
+ unsigned short FMFValue = getFMFValue();
+ return (FMFValue & FastMathFlags::NoNaNs) != 0;
}
/// Test if this operation's arguments and results are assumed not-infinite.
bool hasNoInfs() const {
- return (SubclassOptionalData & FastMathFlags::NoInfs) != 0;
+ unsigned short FMFValue = getFMFValue();
+ return (FMFValue & FastMathFlags::NoInfs) != 0;
}
/// Test if this operation can ignore the sign of zero.
bool hasNoSignedZeros() const {
- return (SubclassOptionalData & FastMathFlags::NoSignedZeros) != 0;
+ unsigned short FMFValue = getFMFValue();
+ return (FMFValue & FastMathFlags::NoSignedZeros) != 0;
}
/// Test if this operation can use reciprocal multiply instead of division.
bool hasAllowReciprocal() const {
- return (SubclassOptionalData & FastMathFlags::AllowReciprocal) != 0;
+ unsigned short FMFValue = getFMFValue();
+ return (FMFValue & FastMathFlags::AllowReciprocal) != 0;
}
/// Test if this operation can be floating-point contracted (FMA).
bool hasAllowContract() const {
- return (SubclassOptionalData & FastMathFlags::AllowContract) != 0;
+ unsigned short FMFValue = getFMFValue();
+ return (FMFValue & FastMathFlags::AllowContract) != 0;
}
/// Test if this operation allows approximations of math library functions or
/// intrinsics.
bool hasApproxFunc() const {
- return (SubclassOptionalData & FastMathFlags::ApproxFunc) != 0;
+ unsigned short FMFValue = getFMFValue();
+ return (FMFValue & FastMathFlags::ApproxFunc) != 0;
}
/// Convenience function for getting all the fast-math flags
FastMathFlags getFastMathFlags() const {
- return FastMathFlags(SubclassOptionalData);
+ unsigned short FMFValue = getFMFValue();
+ return FastMathFlags(FMFValue);
}
/// Get the maximum error permitted by this operation in ULPs. An accuracy of
diff --git a/llvm/lib/IR/Instruction.cpp b/llvm/lib/IR/Instruction.cpp
index 8aa19a436a157..1974634dffbcb 100644
--- a/llvm/lib/IR/Instruction.cpp
+++ b/llvm/lib/IR/Instruction.cpp
@@ -708,7 +708,8 @@ void Instruction::copyFastMathFlags(const Instruction *I) {
copyFastMathFlags(I->getFastMathFlags());
}
-void Instruction::copyIRFlags(const Value *V, bool IncludeWrapFlags) {
+void Instruction::copyIRFlags(const Value *V, bool IncludeWrapFlags,
+ bool IncludeFastMathFlags) {
// Copy the wrapping flags.
if (IncludeWrapFlags && isa<OverflowingBinaryOperator>(this)) {
if (auto *OB = dyn_cast<OverflowingBinaryOperator>(V)) {
@@ -734,9 +735,11 @@ void Instruction::copyIRFlags(const Value *V, bool IncludeWrapFlags) {
DestPD->setIsDisjoint(SrcPD->isDisjoint());
// Copy the fast-math flags.
- if (auto *FP = dyn_cast<FPMathOperator>(V))
- if (isa<FPMathOperator>(this))
- copyFastMathFlags(FP->getFastMathFlags());
+ if (IncludeFastMathFlags) {
+ if (auto *FP = dyn_cast<FPMathOperator>(V))
+ if (isa<FPMathOperator>(this))
+ copyFastMathFlags(FP->getFastMathFlags());
+ }
if (auto *SrcGEP = dyn_cast<GetElementPtrInst>(V))
if (auto *DestGEP = dyn_cast<GetElementPtrInst>(this))
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 8a220c48acac8..2fd8758698b88 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -802,6 +802,7 @@ CallInst::CallInst(const CallInst &CI, AllocInfo AllocInfo)
std::copy(CI.bundle_op_info_begin(), CI.bundle_op_info_end(),
bundle_op_info_begin());
SubclassOptionalData = CI.SubclassOptionalData;
+ FMFValue = CI.FMFValue;
}
CallInst *CallInst::Create(CallInst *CI, ArrayRef<OperandBundleDef> OpB,
@@ -813,6 +814,7 @@ CallInst *CallInst::Create(CallInst *CI, ArrayRef<OperandBundleDef> OpB,
NewCI->setTailCallKind(CI->getTailCallKind());
NewCI->setCallingConv(CI->getCallingConv());
NewC...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/191190
More information about the llvm-commits
mailing list