[llvm-branch-commits] [llvm] Bundle operands to specify denormal modes (PR #136501)
Serge Pavlov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Apr 24 09:04:52 PDT 2025
https://github.com/spavloff updated https://github.com/llvm/llvm-project/pull/136501
>From 22742e24c1eef3ecc0fb4294dac9f42c9d160019 Mon Sep 17 00:00:00 2001
From: Serge Pavlov <sepavloff at gmail.com>
Date: Thu, 17 Apr 2025 18:42:15 +0700
Subject: [PATCH 1/2] Bundle operands to specify denormal modes
Two new operands are now supported in the "fp.control" operand bundle:
* "denorm.in=xxx" - specifies the inpot denormal mode.
* "denorm.out=xxx" - specifies the output denormal mode.
Here xxx must be one of the following values:
* "ieee" - preserve denormals.
* "zero" - flush to zero preserving sign.
* "pzero" - flush to positive zero.
* "dyn" - mode is dynamically read from a control register.
These values align those permitted in the "denormal-fp-math" function
attribute.
---
llvm/docs/LangRef.rst | 18 +-
llvm/include/llvm/ADT/FloatingPointMode.h | 33 ++++
llvm/include/llvm/IR/InstrTypes.h | 21 +++
llvm/lib/Analysis/ConstantFolding.cpp | 24 ++-
llvm/lib/IR/Instructions.cpp | 168 +++++++++++++++++-
llvm/lib/IR/Verifier.cpp | 14 ++
.../constant-fold-fp-denormal-strict.ll | 91 ++++++++++
llvm/test/Verifier/fp-intrinsics.ll | 36 ++++
8 files changed, 394 insertions(+), 11 deletions(-)
create mode 100644 llvm/test/Transforms/InstSimplify/constant-fold-fp-denormal-strict.ll
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 8252971aa8a58..954f0e96b46f6 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -3084,7 +3084,10 @@ floating-point control modes and the treatment of status bits respectively.
An operand bundle tagged with "fp.control" contains information about the
control modes used for the operation execution. Operands specified in this
-bundle represent particular options. Currently, only rounding mode is supported.
+bundle represent particular options. The following modes are supported:
+
+* rounding mode,
+* denormal behavior.
Rounding mode is represented by a metadata string value, which specifies the
the mode used for the operation evaluation. Possible values are:
@@ -3103,6 +3106,19 @@ rounding rounding mode is taken from the control register (dynamic rounding).
In the particular case of :ref:`default floating-point environment <floatenv>`,
the operation uses rounding to nearest, ties to even.
+Denormal behavior defines whether denormal values are flushed to zero during
+the call's execution. This behavior is specified separately for input and
+output values. Such specification is a string, which starts with
+"denorm.in=" or "denorm.out=" respectively. The remainder of the string should
+be one of the values:
+
+::
+
+ ``"ieee"`` - preserve denormals,
+ ``"zero"`` - flush to +0.0 or -0.0 depending on value sign,
+ ``"pzero"`` - flush to +0.0,
+ ``"dyn"`` - concrete mode is read from some register.
+
An operand bundle tagged with "fp.except" may be associated with operations
that can read or write floating-point exception flags. It contains a single
metadata string value, which can have one of the following values:
diff --git a/llvm/include/llvm/ADT/FloatingPointMode.h b/llvm/include/llvm/ADT/FloatingPointMode.h
index 639d931ef88fe..5fceccfd1d0bf 100644
--- a/llvm/include/llvm/ADT/FloatingPointMode.h
+++ b/llvm/include/llvm/ADT/FloatingPointMode.h
@@ -234,6 +234,39 @@ void DenormalMode::print(raw_ostream &OS) const {
OS << denormalModeKindName(Output) << ',' << denormalModeKindName(Input);
}
+/// If the specified string represents denormal mode as used in operand bundles,
+/// returns the corresponding mode.
+inline std::optional<DenormalMode::DenormalModeKind>
+parseDenormalKindFromOperandBundle(StringRef Str) {
+ if (Str == "ieee")
+ return DenormalMode::IEEE;
+ if (Str == "zero")
+ return DenormalMode::PreserveSign;
+ if (Str == "pzero")
+ return DenormalMode::PositiveZero;
+ if (Str == "dyn")
+ return DenormalMode::Dynamic;
+ return std::nullopt;
+}
+
+/// Converts the specified denormal mode into string suitable for use in an
+/// operand bundle.
+inline std::optional<StringRef>
+printDenormalForOperandBundle(DenormalMode::DenormalModeKind Mode) {
+ switch (Mode) {
+ case DenormalMode::IEEE:
+ return "ieee";
+ case DenormalMode::PreserveSign:
+ return "zero";
+ case DenormalMode::PositiveZero:
+ return "pzero";
+ case DenormalMode::Dynamic:
+ return "dyn";
+ default:
+ return std::nullopt;
+ }
+}
+
/// Floating-point class tests, supported by 'is_fpclass' intrinsic. Actual
/// test may be an OR combination of basic tests.
enum FPClassTest : unsigned {
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 8425243e5efe9..8492c911ffc6a 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1092,12 +1092,24 @@ template <typename InputTy> class OperandBundleDefT {
using OperandBundleDef = OperandBundleDefT<Value *>;
using ConstOperandBundleDef = OperandBundleDefT<const Value *>;
+std::optional<StringRef> getBundleOperandByPrefix(OperandBundleUse Bundle,
+ StringRef Prefix);
+void addOperandToBundleTag(LLVMContext &Ctx,
+ SmallVectorImpl<OperandBundleDef> &Bundles,
+ StringRef Tag, size_t PrefixSize, StringRef Val);
+
void addFPRoundingBundle(LLVMContext &Ctx,
SmallVectorImpl<OperandBundleDef> &Bundles,
RoundingMode Rounding);
void addFPExceptionBundle(LLVMContext &Ctx,
SmallVectorImpl<OperandBundleDef> &Bundles,
fp::ExceptionBehavior Except);
+void addFPInputDenormBundle(LLVMContext &Ctx,
+ SmallVectorImpl<OperandBundleDef> &Bundles,
+ DenormalMode::DenormalModeKind Mode);
+void addFPOutputDenormBundle(LLVMContext &Ctx,
+ SmallVectorImpl<OperandBundleDef> &Bundles,
+ DenormalMode::DenormalModeKind Mode);
//===----------------------------------------------------------------------===//
// CallBase Class
@@ -2171,6 +2183,15 @@ class CallBase : public Instruction {
/// Return exception behavior specified for this call.
fp::ExceptionBehavior getExceptionBehavior() const;
+ /// Return input denormal mode specified by operand bundles.
+ DenormalMode::DenormalModeKind getInputDenormMode() const;
+
+ /// Return output denormal mode specified by operand bundles.
+ DenormalMode::DenormalModeKind getOutputDenormMode() const;
+
+ /// Return input and output denormal modes specified by operand bundles.
+ DenormalMode getDenormMode() const;
+
/// Used to keep track of an operand bundle. See the main comment on
/// OperandBundleUser above.
struct BundleOpInfo {
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index dc905ab03e861..602aa380fab1f 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -1354,6 +1354,11 @@ static ConstantFP *flushDenormalConstantFP(ConstantFP *CFP,
if (!APF.isDenormal())
return CFP;
+ if (auto *CB = dyn_cast<CallBase>(Inst)) {
+ auto Mode = IsOutput ? CB->getOutputDenormMode() : CB->getInputDenormMode();
+ return flushDenormalConstant(CFP->getType(), APF, Mode);
+ }
+
DenormalMode Mode = getInstrDenormalMode(Inst, CFP->getType());
return flushDenormalConstant(CFP->getType(), APF,
IsOutput ? Mode.Output : Mode.Input);
@@ -2976,12 +2981,20 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
}
if (const auto *Op1 = dyn_cast<ConstantFP>(Operands[0])) {
- const APFloat &Op1V = Op1->getValueAPF();
+ ConstantFP *Op1F =
+ flushDenormalConstantFP(const_cast<ConstantFP *>(Op1), Call, false);
+ if (!Op1F)
+ return nullptr;
+ const APFloat &Op1V = Op1F->getValueAPF();
if (const auto *Op2 = dyn_cast<ConstantFP>(Operands[1])) {
if (Op2->getType() != Op1->getType())
return nullptr;
- const APFloat &Op2V = Op2->getValueAPF();
+ ConstantFP *Op2F =
+ flushDenormalConstantFP(const_cast<ConstantFP *>(Op2), Call, false);
+ if (!Op2F)
+ return nullptr;
+ const APFloat &Op2V = Op2F->getValueAPF();
if (const auto *ConstrIntr =
dyn_cast_if_present<ConstrainedFPIntrinsic>(Call)) {
@@ -3011,8 +3024,11 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
return evaluateCompare(Op1V, Op2V, ConstrIntr);
}
if (mayFoldConstrained(const_cast<ConstrainedFPIntrinsic *>(ConstrIntr),
- St))
- return ConstantFP::get(Ty->getContext(), Res);
+ St)) {
+ auto DenormOut = Call->getOutputDenormMode();
+ DenormalMode::DenormalModeKind Mode = DenormOut;
+ return flushDenormalConstant(Op2->getType(), Res, Mode);
+ }
return nullptr;
}
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 1073c2939a8a3..da766473f5fff 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -628,13 +628,17 @@ bool CallBase::hasClobberingOperandBundles() const {
RoundingMode CallBase::getRoundingMode() const {
// Try reading rounding mode from FP bundle.
std::optional<RoundingMode> RM;
- if (auto RoundingBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
- Value *V = RoundingBundle->Inputs.front();
- Metadata *MD = cast<MetadataAsValue>(V)->getMetadata();
- RM = convertBundleToRoundingMode(cast<MDString>(MD)->getString());
+ if (auto ControlBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
+ for (auto &U : ControlBundle->Inputs) {
+ Value *V = U.get();
+ if (auto *MDV = dyn_cast<MetadataAsValue>(V)) {
+ Metadata *MD = MDV->getMetadata();
+ if (auto *MDS = dyn_cast<MDString>(MD))
+ if (auto RM = convertBundleToRoundingMode(MDS->getString()))
+ return *RM;
+ }
+ }
}
- if (RM)
- return *RM;
// If this is a constrained intrinsic, get rounding mode from its metadata
// arguments.
@@ -678,6 +682,71 @@ fp::ExceptionBehavior CallBase::getExceptionBehavior() const {
return fp::ebIgnore;
}
+DenormalMode::DenormalModeKind CallBase::getInputDenormMode() const {
+ if (auto InDenormBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
+ auto DenormOperand =
+ getBundleOperandByPrefix(*InDenormBundle, "denorm.in=");
+ if (DenormOperand) {
+ if (auto Mode = parseDenormalKindFromOperandBundle(*DenormOperand))
+ return *Mode;
+ } else {
+ return DenormalMode::IEEE;
+ }
+ }
+
+ if (!getParent())
+ return DenormalMode::IEEE;
+ const Function *F = getFunction();
+ if (!F)
+ return DenormalMode::IEEE;
+
+ Type *Ty = nullptr;
+ for (auto &A : args())
+ if (auto *T = A.get()->getType(); T->isFPOrFPVectorTy()) {
+ Ty = T;
+ break;
+ }
+ assert(Ty && "Some input argument must be of floating-point type");
+
+ Ty = Ty->getScalarType();
+ return F->getDenormalMode(Ty->getFltSemantics()).Input;
+}
+
+DenormalMode::DenormalModeKind CallBase::getOutputDenormMode() const {
+ if (auto InDenormBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
+ auto DenormOperand =
+ getBundleOperandByPrefix(*InDenormBundle, "denorm.out=");
+ if (DenormOperand) {
+ if (auto Mode = parseDenormalKindFromOperandBundle(*DenormOperand))
+ return *Mode;
+ } else {
+ return DenormalMode::IEEE;
+ }
+ }
+
+ if (!getParent())
+ return DenormalMode::IEEE;
+ const Function *F = getFunction();
+ if (!F)
+ return DenormalMode::IEEE;
+
+ Type *Ty = getType();
+ assert(Ty->isFPOrFPVectorTy() && "Unexpected output type");
+
+ Ty = Ty->getScalarType();
+ return F->getDenormalMode(Ty->getFltSemantics()).Output;
+}
+
+DenormalMode CallBase::getDenormMode() const {
+ auto InputMode = getInputDenormMode();
+ auto OutputMode = getOutputDenormMode();
+ if (!InputMode)
+ InputMode = DenormalMode::IEEE;
+ if (!OutputMode)
+ OutputMode = DenormalMode::IEEE;
+ return DenormalMode(OutputMode, InputMode);
+}
+
MemoryEffects CallBase::getFloatingPointMemoryEffects() const {
if (Intrinsic::ID IntrID = getIntrinsicID())
if (const BasicBlock *BB = getParent())
@@ -794,6 +863,69 @@ bool CallBase::hasArgumentWithAdditionalReturnCaptureComponents() const {
return false;
}
+std::optional<StringRef> llvm::getBundleOperandByPrefix(OperandBundleUse Bundle,
+ StringRef Prefix) {
+ for (const auto &Item : Bundle.Inputs) {
+ Metadata *MD = cast<MetadataAsValue>(Item.get())->getMetadata();
+ if (const auto *MDS = dyn_cast<MDString>(MD)) {
+ StringRef Str = MDS->getString();
+ if (Str.consume_front(Prefix))
+ return Str;
+ }
+ }
+ return std::nullopt;
+}
+
+void llvm::addOperandToBundleTag(LLVMContext &Ctx,
+ SmallVectorImpl<OperandBundleDef> &Bundles,
+ StringRef Tag, size_t PrefixSize,
+ StringRef Val) {
+ assert(PrefixSize > 0 && "Unexpected prefix size");
+ assert(PrefixSize < Val.size() && "Invalid prefix size");
+ StringRef Prefix = Val.take_front(PrefixSize);
+
+ // Find a bundle with the specified tag.
+ OperandBundleDef *Bundle = nullptr;
+ for (OperandBundleDef &OB : Bundles) {
+ if (OB.getTag() == Tag) {
+ Bundle = &OB;
+ break;
+ }
+ }
+
+ // If no such bundle are found, create new one with the single value.
+ if (!Bundle) {
+ auto *MStr = MDString::get(Ctx, Val);
+ auto *MD = MetadataAsValue::get(Ctx, MStr);
+ SmallVector<Value *, 1> BundleValues(1, MD);
+ Bundles.emplace_back(Tag.str(), BundleValues);
+ return;
+ }
+
+ // If the bundle is found, search its values for those started with the given
+ // prefix.
+ SmallVector<Value *, 4> Values(Bundle->inputs());
+ for (auto i = Values.begin(), e = Values.end(); i != e; ++i) {
+ if (auto *MV = dyn_cast<MetadataAsValue>(*i)) {
+ if (auto *MD = dyn_cast<MDString>(MV->getMetadata())) {
+ StringRef Str = MD->getString();
+ if (Str == Val)
+ // Already in the values.
+ return;
+ if (Str.starts_with(Prefix)) {
+ Values.erase(i);
+ break;
+ }
+ }
+ }
+ }
+
+ auto *ValMD = MDString::get(Ctx, Val);
+ auto *MD = MetadataAsValue::get(Ctx, ValMD);
+ Values.push_back(MD);
+ Bundles.emplace_back("fp.control", Values);
+}
+
void llvm::addFPRoundingBundle(LLVMContext &Ctx,
SmallVectorImpl<OperandBundleDef> &Bundles,
RoundingMode Rounding) {
@@ -814,6 +946,30 @@ void llvm::addFPExceptionBundle(LLVMContext &Ctx,
Bundles.emplace_back("fp.except", EB);
}
+void llvm::addFPInputDenormBundle(LLVMContext &Ctx,
+ SmallVectorImpl<OperandBundleDef> &Bundles,
+ DenormalMode::DenormalModeKind Mode) {
+ std::optional<StringRef> DenormValue = printDenormalForOperandBundle(Mode);
+ if (!DenormValue)
+ return;
+ std::string Prefix = "denorm.in=";
+ std::string DenormItem = Prefix + DenormValue->str();
+
+ addOperandToBundleTag(Ctx, Bundles, "fp.control", Prefix.size(), DenormItem);
+}
+
+void llvm::addFPOutputDenormBundle(LLVMContext &Ctx,
+ SmallVectorImpl<OperandBundleDef> &Bundles,
+ DenormalMode::DenormalModeKind Mode) {
+ std::optional<StringRef> DenormValue = printDenormalForOperandBundle(Mode);
+ if (!DenormValue)
+ return;
+ std::string Prefix = "denorm.out=";
+ std::string DenormItem = Prefix + DenormValue->str();
+
+ addOperandToBundleTag(Ctx, Bundles, "fp.control", Prefix.size(), DenormItem);
+}
+
//===----------------------------------------------------------------------===//
// CallInst Implementation
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 6b7644fef9e71..bb40c4b7b7b83 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -3833,6 +3833,8 @@ void Verifier::visitCallBase(CallBase &Call) {
Check(!FoundFpeControlBundle, "Multiple \"fp.control\" operand bundles",
Call);
bool FoundRoundingMode = false;
+ bool FoundInDenormalMode = false;
+ bool FoundOutDenormalMode = false;
for (auto &U : BU.Inputs) {
Value *V = U.get();
Check(isa<MetadataAsValue>(V),
@@ -3847,6 +3849,18 @@ void Verifier::visitCallBase(CallBase &Call) {
Check(!FoundRoundingMode, "Rounding mode is specified more that once",
Call);
FoundRoundingMode = true;
+ } else if (Item.consume_front("denorm.in=")) {
+ Check(!FoundInDenormalMode,
+ "Input denormal mode is specified more that once", Call);
+ FoundInDenormalMode = true;
+ Check(parseDenormalKindFromOperandBundle(Item),
+ "Invalid input denormal mode", Call);
+ } else if (Item.consume_front("denorm.out=")) {
+ Check(!FoundOutDenormalMode,
+ "Output denormal mode is specified more that once", Call);
+ FoundOutDenormalMode = true;
+ Check(parseDenormalKindFromOperandBundle(Item),
+ "Invalid output denormal mode", Call);
} else {
CheckFailed("Unrecognized value in \"fp.control\" bundle operand",
Call);
diff --git a/llvm/test/Transforms/InstSimplify/constant-fold-fp-denormal-strict.ll b/llvm/test/Transforms/InstSimplify/constant-fold-fp-denormal-strict.ll
new file mode 100644
index 0000000000000..c9886928ce4a4
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/constant-fold-fp-denormal-strict.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instsimplify < %s | FileCheck %s
+
+; 0xB810000000000000 = -0x1.0p-126
+; 0x3800000000000000 = 0x1.0p-127 denormal
+
+define float @test_float_fadd_ieee_strict() #0 {
+; CHECK-LABEL: define float @test_float_fadd_ieee_strict(
+; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: ret float 0xB800000000000000
+;
+ %result = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore")
+ ret float %result
+}
+
+define float @test_float_fadd_strict_ieee() #0 {
+; CHECK-LABEL: define float @test_float_fadd_strict_ieee(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: ret float 0xB800000000000000
+;
+ %result = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore") [ "fp.control" (metadata !"denorm.in=ieee", metadata !"denorm.out=ieee") ]
+ ret float %result
+}
+
+define float @test_float_fadd_strict_inzero() #0 {
+; CHECK-LABEL: define float @test_float_fadd_strict_inzero(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: ret float 0xB810000000000000
+;
+ %result = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore") [ "fp.control" (metadata !"denorm.in=zero", metadata !"denorm.out=ieee") ]
+ ret float %result
+}
+
+define float @test_float_fadd_strict_inpzero() #0 {
+; CHECK-LABEL: define float @test_float_fadd_strict_inpzero(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: ret float 0xB810000000000000
+;
+ %result = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore") [ "fp.control" (metadata !"denorm.in=pzero", metadata !"denorm.out=ieee") ]
+ ret float %result
+}
+
+define float @test_float_fadd_strict_indyn() #0 {
+; CHECK-LABEL: define float @test_float_fadd_strict_indyn(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[RESULT:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore") [ "fp.control"(metadata !"denorm.in=dyn", metadata !"denorm.out=ieee") ]
+; CHECK-NEXT: ret float [[RESULT]]
+;
+ %result = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore") [ "fp.control" (metadata !"denorm.in=dyn", metadata !"denorm.out=ieee") ]
+ ret float %result
+}
+
+define float @test_float_fadd_strict_ieee_outzero() #0 {
+; CHECK-LABEL: define float @test_float_fadd_strict_ieee_outzero(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: ret float -0.000000e+00
+;
+ %result = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore") [ "fp.control" (metadata !"denorm.in=ieee", metadata !"denorm.out=zero") ]
+ ret float %result
+}
+
+define float @test_float_fadd_strict_ieee_outpzero() #0 {
+; CHECK-LABEL: define float @test_float_fadd_strict_ieee_outpzero(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %result = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore") [ "fp.control" (metadata !"denorm.in=ieee", metadata !"denorm.out=pzero") ]
+ ret float %result
+}
+
+define float @test_float_fadd_strict_ieee_outdyn() #0 {
+; CHECK-LABEL: define float @test_float_fadd_strict_ieee_outdyn(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: [[RESULT:%.*]] = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore") [ "fp.control"(metadata !"denorm.in=ieee", metadata !"denorm.out=dyn") ]
+; CHECK-NEXT: ret float [[RESULT]]
+;
+ %result = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore") [ "fp.control" (metadata !"denorm.in=ieee", metadata !"denorm.out=dyn") ]
+ ret float %result
+}
+
+define float @test_float_fadd_strict_zero_outdef() #0 {
+; CHECK-LABEL: define float @test_float_fadd_strict_zero_outdef(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT: ret float 0xB800000000000000
+;
+ %result = call float @llvm.experimental.constrained.fadd.f32(float 0xB810000000000000, float 0x3800000000000000, metadata !"round.towardzero", metadata !"fpexcept.ignore") [ "fp.control" (metadata !"denorm.in=ieee") ]
+ ret float %result
+}
+
+attributes #0 = { nounwind strictfp "denormal-fp-math"="ieee,ieee" }
+attributes #5 = { nounwind "denormal-fp-math"="ieee,ieee" "denormal-fp-math-f32"="positive-zero,ieee" }
diff --git a/llvm/test/Verifier/fp-intrinsics.ll b/llvm/test/Verifier/fp-intrinsics.ll
index 1e1df1b0da96a..cf67f9b54be92 100644
--- a/llvm/test/Verifier/fp-intrinsics.ll
+++ b/llvm/test/Verifier/fp-intrinsics.ll
@@ -141,5 +141,41 @@ entry:
ret double %ftrunc
}
+; Test fp.control bundle specifies two input demormal modes.
+; CHECK-NEXT: Input denormal mode is specified more that once
+; CHECK-NEXT: %ftrunc = call double @llvm.trunc.f64(double %a) #{{[0-9]+}} [ "fp.control"(metadata !"denorm.in=ieee", metadata !"denorm.in=ieee") ]
+define double @f16(double %a) #0 {
+entry:
+ %ftrunc = call double @llvm.trunc.f64(double %a) #0 [ "fp.control"(metadata !"denorm.in=ieee", metadata !"denorm.in=ieee") ]
+ ret double %ftrunc
+}
+
+; Test fp.control bundle specifies two output demormal modes.
+; CHECK-NEXT: Output denormal mode is specified more that once
+; CHECK-NEXT: %ftrunc = call double @llvm.trunc.f64(double %a) #{{[0-9]+}} [ "fp.control"(metadata !"denorm.out=ieee", metadata !"denorm.out=ieee") ]
+define double @f17(double %a) #0 {
+entry:
+ %ftrunc = call double @llvm.trunc.f64(double %a) #0 [ "fp.control"(metadata !"denorm.out=ieee", metadata !"denorm.out=ieee") ]
+ ret double %ftrunc
+}
+
+; Test fp.control bundle specifies invalid input demormal modes.
+; CHECK-NEXT: Invalid input denormal mode
+; CHECK-NEXT: %ftrunc = call double @llvm.trunc.f64(double %a) #{{[0-9]+}} [ "fp.control"(metadata !"denorm.in=qqq") ]
+define double @f18(double %a) #0 {
+entry:
+ %ftrunc = call double @llvm.trunc.f64(double %a) #0 [ "fp.control"(metadata !"denorm.in=qqq") ]
+ ret double %ftrunc
+}
+
+; Test fp.control bundle specifies invalid output demormal modes.
+; CHECK-NEXT: Invalid output denormal mode
+; CHECK-NEXT: %ftrunc = call double @llvm.trunc.f64(double %a) #{{[0-9]+}} [ "fp.control"(metadata !"denorm.out=qqq") ]
+define double @f19(double %a) #0 {
+entry:
+ %ftrunc = call double @llvm.trunc.f64(double %a) #0 [ "fp.control"(metadata !"denorm.out=qqq") ]
+ ret double %ftrunc
+}
+
attributes #0 = { strictfp }
>From 0e19500df2f470fbcc081abb2889fc9870149bee Mon Sep 17 00:00:00 2001
From: Serge Pavlov <sepavloff at gmail.com>
Date: Thu, 24 Apr 2025 23:00:55 +0700
Subject: [PATCH 2/2] Address review comments
- introduced bundle operands for 32-bit vslues,
- simplified demormal mode search,
- updated documentation.
---
llvm/docs/LangRef.rst | 23 +++-
llvm/include/llvm/IR/InstrTypes.h | 30 +++--
llvm/lib/Analysis/ConstantFolding.cpp | 3 +-
llvm/lib/IR/Instructions.cpp | 162 +++++++++++++++++---------
4 files changed, 146 insertions(+), 72 deletions(-)
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 954f0e96b46f6..c5d31e66e0686 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -3114,10 +3114,25 @@ be one of the values:
::
- ``"ieee"`` - preserve denormals,
- ``"zero"`` - flush to +0.0 or -0.0 depending on value sign,
- ``"pzero"`` - flush to +0.0,
- ``"dyn"`` - concrete mode is read from some register.
+ "ieee" - preserve denormals,
+ "zero" - flush to +0.0 or -0.0 depending on value sign,
+ "pzero" - flush to +0.0,
+ "dyn" - concrete mode is read from some register.
+
+Such bundle operand specifies denormal behavior for all floating-point types.
+It is possible to override denormal behavior for specific type, if the target
+supports that. Now only type "f32" allows such overriding, if a bundle operand
+has prefix "denorm.f32.in=" or "denormal.f32.out=", its specifies denormal mode
+for ``float`` values in the affected instruction. For example:
+
+.. code-block:: llvm
+
+ ; denormal arguments are flushed to zero preserving sign.
+ call float @llvm.trunc.f32(float %x) [ "fp.control"(metadata !"denorm.in=ieee", metadata !"denorm.f32.in=zero") ]
+
+ ; denormal arguments are flushed to +0.0.
+ call float @llvm.trunc.f32(float %x) [ "fp.control"(metadata !"denorm.in=pzero") ]
+
An operand bundle tagged with "fp.except" may be associated with operations
that can read or write floating-point exception flags. It contains a single
diff --git a/llvm/include/llvm/IR/InstrTypes.h b/llvm/include/llvm/IR/InstrTypes.h
index 8492c911ffc6a..1192318d57784 100644
--- a/llvm/include/llvm/IR/InstrTypes.h
+++ b/llvm/include/llvm/IR/InstrTypes.h
@@ -1104,12 +1104,15 @@ void addFPRoundingBundle(LLVMContext &Ctx,
void addFPExceptionBundle(LLVMContext &Ctx,
SmallVectorImpl<OperandBundleDef> &Bundles,
fp::ExceptionBehavior Except);
-void addFPInputDenormBundle(LLVMContext &Ctx,
- SmallVectorImpl<OperandBundleDef> &Bundles,
- DenormalMode::DenormalModeKind Mode);
-void addFPOutputDenormBundle(LLVMContext &Ctx,
- SmallVectorImpl<OperandBundleDef> &Bundles,
- DenormalMode::DenormalModeKind Mode);
+std::optional<DenormalMode::DenormalModeKind>
+getDenormModeBundle(const OperandBundleUse &Control, bool Unput,
+ const fltSemantics *FPSem);
+void addInputDenormBundle(LLVMContext &Ctx,
+ SmallVectorImpl<OperandBundleDef> &Bundles,
+ DenormalMode::DenormalModeKind Mode);
+void addOutputDenormBundle(LLVMContext &Ctx,
+ SmallVectorImpl<OperandBundleDef> &Bundles,
+ DenormalMode::DenormalModeKind Mode);
//===----------------------------------------------------------------------===//
// CallBase Class
@@ -2184,13 +2187,20 @@ class CallBase : public Instruction {
fp::ExceptionBehavior getExceptionBehavior() const;
/// Return input denormal mode specified by operand bundles.
- DenormalMode::DenormalModeKind getInputDenormMode() const;
+ std::optional<DenormalMode::DenormalModeKind>
+ getInputDenormMode(const fltSemantics *FPSem = nullptr) const;
/// Return output denormal mode specified by operand bundles.
- DenormalMode::DenormalModeKind getOutputDenormMode() const;
+ std::optional<DenormalMode::DenormalModeKind>
+ getOutputDenormMode(const fltSemantics *FPSem = nullptr) const;
- /// Return input and output denormal modes specified by operand bundles.
- DenormalMode getDenormMode() const;
+ /// Return input denormal mode specified by operand bundles.
+ std::optional<DenormalMode::DenormalModeKind>
+ getInputDenormModeFromBundle(const fltSemantics *FPSem = nullptr) const;
+
+ /// Return output denormal mode specified by operand bundles.
+ std::optional<DenormalMode::DenormalModeKind>
+ getOutputDenormModeFromBundle(const fltSemantics *FPSem = nullptr) const;
/// Used to keep track of an operand bundle. See the main comment on
/// OperandBundleUser above.
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 602aa380fab1f..2823b8034fea4 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -3025,8 +3025,7 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
}
if (mayFoldConstrained(const_cast<ConstrainedFPIntrinsic *>(ConstrIntr),
St)) {
- auto DenormOut = Call->getOutputDenormMode();
- DenormalMode::DenormalModeKind Mode = DenormOut;
+ DenormalMode::DenormalModeKind Mode = *Call->getOutputDenormMode();
return flushDenormalConstant(Op2->getType(), Res, Mode);
}
return nullptr;
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index da766473f5fff..1d7fed3b5ffb6 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -682,69 +682,96 @@ fp::ExceptionBehavior CallBase::getExceptionBehavior() const {
return fp::ebIgnore;
}
-DenormalMode::DenormalModeKind CallBase::getInputDenormMode() const {
- if (auto InDenormBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
- auto DenormOperand =
- getBundleOperandByPrefix(*InDenormBundle, "denorm.in=");
- if (DenormOperand) {
- if (auto Mode = parseDenormalKindFromOperandBundle(*DenormOperand))
- return *Mode;
- } else {
- return DenormalMode::IEEE;
- }
+/// Returns the input denormal behavior specified by operand bundles.
+///
+/// Searches for the bundle operand that specifies denormal behavior for the
+/// given type, or for any type if the argument is \a nullptr.
+///
+/// \param FPSem - pointer to the FP semantics of a call argument type. If
+/// \a nullptr, searches for a common denormal mode
+/// specification, specified by the operand with prefix
+/// "denorm.in=".
+std::optional<DenormalMode::DenormalModeKind>
+CallBase::getInputDenormModeFromBundle(const fltSemantics *FPSem) const {
+ if (auto ControlBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
+ if (FPSem)
+ if (auto Result = getDenormModeBundle(*ControlBundle, true, FPSem))
+ return Result;
+ if (auto Result = getDenormModeBundle(*ControlBundle, true, nullptr))
+ return Result;
+ }
+ return std::nullopt;
+}
+
+/// Returns the output denormal behavior specified by operand bundles.
+///
+/// Searches for the bundle operand that specifies denormal behavior for the
+/// given type, or for any type if the argument is \a nullptr.
+///
+/// \param FPSem - pointer to the FP semantics of the call result type. If
+/// \a nullptr, searches for a common denormal mode
+/// specification, specified by the operand with prefix
+/// "denorm.out=".
+std::optional<DenormalMode::DenormalModeKind>
+CallBase::getOutputDenormModeFromBundle(const fltSemantics *FPSem) const {
+ if (auto ControlBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
+ if (FPSem)
+ if (auto Result = getDenormModeBundle(*ControlBundle, false, FPSem))
+ return Result;
+ if (auto Result = getDenormModeBundle(*ControlBundle, false, nullptr))
+ return Result;
}
+ return std::nullopt;
+}
+
+/// Returns the input denormal behavior to be used for the call evaluation.
+///
+/// Searches operand bundles first for matching denormal behavior
+/// specifications. If not found in bundles, falls back to checking function
+/// attributes.
+///
+/// \param FPSem - pointer to the FP semantics of the call argument type. If
+/// \a nullptr, searches for a common denormal mode
+/// specification, using the operand with prefix "denorm.in=".
+std::optional<DenormalMode::DenormalModeKind>
+CallBase::getInputDenormMode(const fltSemantics *FPSem) const {
+ if (auto Result = getInputDenormModeFromBundle(FPSem))
+ return Result;
if (!getParent())
- return DenormalMode::IEEE;
+ return std::nullopt;
const Function *F = getFunction();
if (!F)
- return DenormalMode::IEEE;
-
- Type *Ty = nullptr;
- for (auto &A : args())
- if (auto *T = A.get()->getType(); T->isFPOrFPVectorTy()) {
- Ty = T;
- break;
- }
- assert(Ty && "Some input argument must be of floating-point type");
+ return std::nullopt;
- Ty = Ty->getScalarType();
- return F->getDenormalMode(Ty->getFltSemantics()).Input;
+ if (FPSem)
+ return F->getDenormalMode(*FPSem).Input;
+ return F->getDenormalModeRaw().Input;
}
-DenormalMode::DenormalModeKind CallBase::getOutputDenormMode() const {
- if (auto InDenormBundle = getOperandBundle(LLVMContext::OB_fp_control)) {
- auto DenormOperand =
- getBundleOperandByPrefix(*InDenormBundle, "denorm.out=");
- if (DenormOperand) {
- if (auto Mode = parseDenormalKindFromOperandBundle(*DenormOperand))
- return *Mode;
- } else {
- return DenormalMode::IEEE;
- }
- }
+/// Returns the output denormal behavior to be used for the call evaluation.
+///
+/// Searches operand bundles first for matching denormal behavior
+/// specifications. If not found in bundles, falls back to checking function
+/// attributes.
+///
+/// \param FPSem - pointer to the FP semantics of the call result type. If
+/// \a nullptr, searches for a common denormal mode
+/// specification, using the operand with prefix "denorm.out=".
+std::optional<DenormalMode::DenormalModeKind>
+CallBase::getOutputDenormMode(const fltSemantics *FPSem) const {
+ if (auto Result = getOutputDenormModeFromBundle(FPSem))
+ return Result;
if (!getParent())
- return DenormalMode::IEEE;
+ return std::nullopt;
const Function *F = getFunction();
if (!F)
- return DenormalMode::IEEE;
-
- Type *Ty = getType();
- assert(Ty->isFPOrFPVectorTy() && "Unexpected output type");
-
- Ty = Ty->getScalarType();
- return F->getDenormalMode(Ty->getFltSemantics()).Output;
-}
+ return std::nullopt;
-DenormalMode CallBase::getDenormMode() const {
- auto InputMode = getInputDenormMode();
- auto OutputMode = getOutputDenormMode();
- if (!InputMode)
- InputMode = DenormalMode::IEEE;
- if (!OutputMode)
- OutputMode = DenormalMode::IEEE;
- return DenormalMode(OutputMode, InputMode);
+ if (FPSem)
+ return F->getDenormalMode(*FPSem).Output;
+ return F->getDenormalModeRaw().Output;
}
MemoryEffects CallBase::getFloatingPointMemoryEffects() const {
@@ -946,9 +973,32 @@ void llvm::addFPExceptionBundle(LLVMContext &Ctx,
Bundles.emplace_back("fp.except", EB);
}
-void llvm::addFPInputDenormBundle(LLVMContext &Ctx,
- SmallVectorImpl<OperandBundleDef> &Bundles,
- DenormalMode::DenormalModeKind Mode) {
+static StringRef getBundledDenormPrefix(const fltSemantics *FPSem, bool Input) {
+ // Only f32 is supported now.
+ unsigned TypeIndex = FPSem == &APFloat::IEEEsingle();
+ if (FPSem && !TypeIndex)
+ return StringRef();
+ static const StringRef Prefix[2][2] = {{"denorm.out=", "denorm.in="},
+ {"denorm.f32.out=", "denorm.f32.in="}};
+ return Prefix[TypeIndex][Input];
+}
+
+std::optional<DenormalMode::DenormalModeKind>
+llvm::getDenormModeBundle(const OperandBundleUse &Control, bool Unput,
+ const fltSemantics *FPSem) {
+ assert(Control.getTagID() == LLVMContext::OB_fp_control);
+ StringRef Prefix = getBundledDenormPrefix(FPSem, true);
+ if (Prefix.empty())
+ return std::nullopt;
+ auto DenormOperand = getBundleOperandByPrefix(Control, Prefix);
+ if (DenormOperand)
+ return parseDenormalKindFromOperandBundle(*DenormOperand);
+ return std::nullopt;
+}
+
+void llvm::addInputDenormBundle(LLVMContext &Ctx,
+ SmallVectorImpl<OperandBundleDef> &Bundles,
+ DenormalMode::DenormalModeKind Mode) {
std::optional<StringRef> DenormValue = printDenormalForOperandBundle(Mode);
if (!DenormValue)
return;
@@ -958,9 +1008,9 @@ void llvm::addFPInputDenormBundle(LLVMContext &Ctx,
addOperandToBundleTag(Ctx, Bundles, "fp.control", Prefix.size(), DenormItem);
}
-void llvm::addFPOutputDenormBundle(LLVMContext &Ctx,
- SmallVectorImpl<OperandBundleDef> &Bundles,
- DenormalMode::DenormalModeKind Mode) {
+void llvm::addOutputDenormBundle(LLVMContext &Ctx,
+ SmallVectorImpl<OperandBundleDef> &Bundles,
+ DenormalMode::DenormalModeKind Mode) {
std::optional<StringRef> DenormValue = printDenormalForOperandBundle(Mode);
if (!DenormValue)
return;
More information about the llvm-branch-commits
mailing list