[llvm] [IRBuilder] Fold binary intrinsics (PR #80743)
Artem Tyurin via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 22 05:49:22 PST 2024
https://github.com/agentcooper updated https://github.com/llvm/llvm-project/pull/80743
>From 84a1fd55bb77d967e6a9195388f763d4af8e58f2 Mon Sep 17 00:00:00 2001
From: Artem Tyurin <artem.tyurin at gmail.com>
Date: Mon, 5 Feb 2024 22:25:43 +0100
Subject: [PATCH 1/7] [IRBuilder] Fold binary intrinsics
Fixes https://github.com/llvm/llvm-project/issues/61240.
---
.../llvm/Analysis/InstSimplifyFolder.h | 6 ++++++
llvm/include/llvm/Analysis/TargetFolder.h | 5 +++++
llvm/include/llvm/IR/ConstantFolder.h | 20 +++++++++++++++++++
llvm/include/llvm/IR/IRBuilderFolder.h | 3 +++
llvm/include/llvm/IR/NoFolder.h | 4 ++++
llvm/lib/IR/IRBuilder.cpp | 2 ++
.../Transforms/Vectorize/SLPVectorizer.cpp | 8 --------
7 files changed, 40 insertions(+), 8 deletions(-)
diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
index 23e2ea80e8cbe6..d8ec313b494677 100644
--- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h
+++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
@@ -117,6 +117,12 @@ class InstSimplifyFolder final : public IRBuilderFolder {
return simplifyCastInst(Op, V, DestTy, SQ);
}
+ Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS,
+ Value *RHS) const override {
+ // TODO: should this be defined?
+ return nullptr;
+ }
+
//===--------------------------------------------------------------------===//
// Cast/Conversion Operators
//===--------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/Analysis/TargetFolder.h b/llvm/include/llvm/Analysis/TargetFolder.h
index 978e1002515fc0..cabce12541435d 100644
--- a/llvm/include/llvm/Analysis/TargetFolder.h
+++ b/llvm/include/llvm/Analysis/TargetFolder.h
@@ -191,6 +191,11 @@ class TargetFolder final : public IRBuilderFolder {
return nullptr;
}
+ Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS, Value *RHS) const override {
+ // TODO: should this be defined?
+ return nullptr;
+ }
+
//===--------------------------------------------------------------------===//
// Cast/Conversion Operators
//===--------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/ConstantFolder.h b/llvm/include/llvm/IR/ConstantFolder.h
index c2b30a65e32e25..001d7b1fdf44e8 100644
--- a/llvm/include/llvm/IR/ConstantFolder.h
+++ b/llvm/include/llvm/IR/ConstantFolder.h
@@ -183,6 +183,26 @@ class ConstantFolder final : public IRBuilderFolder {
return nullptr;
}
+ Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS,
+ Value *RHS) const override {
+ auto *LC = dyn_cast<Constant>(LHS);
+ auto *RC = dyn_cast<Constant>(RHS);
+ if (LC && RC) {
+ if (ID == Intrinsic::maxnum) {
+ return ConstantFP::get(LHS->getType(),
+ maxnum(cast<ConstantFP>(LHS)->getValueAPF(),
+ cast<ConstantFP>(RHS)->getValueAPF()));
+ }
+ if (ID == Intrinsic::minnum) {
+ return ConstantFP::get(LHS->getType(),
+ minnum(cast<ConstantFP>(LHS)->getValueAPF(),
+ cast<ConstantFP>(RHS)->getValueAPF()));
+ }
+ // TODO: use switch, handle more intrinsics
+ }
+ return nullptr;
+ }
+
//===--------------------------------------------------------------------===//
// Cast/Conversion Operators
//===--------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/IRBuilderFolder.h b/llvm/include/llvm/IR/IRBuilderFolder.h
index bd2324dfc5f1ba..3e0fd093b3f66f 100644
--- a/llvm/include/llvm/IR/IRBuilderFolder.h
+++ b/llvm/include/llvm/IR/IRBuilderFolder.h
@@ -73,6 +73,9 @@ class IRBuilderFolder {
virtual Value *FoldCast(Instruction::CastOps Op, Value *V,
Type *DestTy) const = 0;
+ virtual Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS,
+ Value *RHS) const = 0;
+
//===--------------------------------------------------------------------===//
// Cast/Conversion Operators
//===--------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/NoFolder.h b/llvm/include/llvm/IR/NoFolder.h
index a612f98465aeaa..787dd10b4cbaac 100644
--- a/llvm/include/llvm/IR/NoFolder.h
+++ b/llvm/include/llvm/IR/NoFolder.h
@@ -112,6 +112,10 @@ class NoFolder final : public IRBuilderFolder {
return nullptr;
}
+ Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS, Value *RHS) const override {
+ return nullptr;
+ }
+
//===--------------------------------------------------------------------===//
// Cast/Conversion Operators
//===--------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index b09b80f95871a1..e5015c1368ceae 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -922,6 +922,8 @@ CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
Value *RHS,
Instruction *FMFSource,
const Twine &Name) {
+ if (Value *V = Folder.FoldBinaryIntrinsics(ID, LHS, RHS))
+ return (CallInst *) V; // TODO: should return value be changed to Value *?
Module *M = BB->getModule();
Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b8d04322de2984..b88f6d060e28ef 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14071,16 +14071,8 @@ class HorizontalReduction {
return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
Name);
case RecurKind::FMax:
- if (IsConstant)
- return ConstantFP::get(LHS->getType(),
- maxnum(cast<ConstantFP>(LHS)->getValueAPF(),
- cast<ConstantFP>(RHS)->getValueAPF()));
return Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS);
case RecurKind::FMin:
- if (IsConstant)
- return ConstantFP::get(LHS->getType(),
- minnum(cast<ConstantFP>(LHS)->getValueAPF(),
- cast<ConstantFP>(RHS)->getValueAPF()));
return Builder.CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS);
case RecurKind::FMaximum:
if (IsConstant)
>From 4044d93d4b6333a6d7dad24eff2e1de7a9f9d302 Mon Sep 17 00:00:00 2001
From: Artem Tyurin <artem.tyurin at gmail.com>
Date: Tue, 6 Feb 2024 17:55:46 +0100
Subject: [PATCH 2/7] Reuse simplifyBinaryIntrinsic
---
llvm/include/llvm/Analysis/InstSimplifyFolder.h | 3 +--
llvm/include/llvm/Analysis/InstructionSimplify.h | 5 +++++
llvm/lib/Analysis/InstructionSimplify.cpp | 8 ++------
3 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
index d8ec313b494677..fbcdf79927d42f 100644
--- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h
+++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
@@ -119,8 +119,7 @@ class InstSimplifyFolder final : public IRBuilderFolder {
Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS,
Value *RHS) const override {
- // TODO: should this be defined?
- return nullptr;
+ return simplifyBinaryIntrinsic(ID, LHS->getType(), LHS, RHS, SQ, nullptr);
}
//===--------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h
index a29955a06cf4e0..c48812c7923d65 100644
--- a/llvm/include/llvm/Analysis/InstructionSimplify.h
+++ b/llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -186,6 +186,11 @@ Value *simplifyExtractElementInst(Value *Vec, Value *Idx,
Value *simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty,
const SimplifyQuery &Q);
+/// Given operands for a BinaryIntrinsic, fold the result or return null.
+Value *simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType, Value *Op0, Value *Op1,
+ const SimplifyQuery &Q,
+ const CallBase *Call);
+
/// Given operands for a ShuffleVectorInst, fold the result or return null.
/// See class ShuffleVectorInst for a description of the mask representation.
Value *simplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef<int> Mask,
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index c10b9d9be101b3..ee70b9fc2d0ae4 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -6379,11 +6379,9 @@ static Value *foldMinimumMaximumSharedOp(Intrinsic::ID IID, Value *Op0,
return nullptr;
}
-static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
+Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType, Value *Op0, Value *Op1,
const SimplifyQuery &Q,
const CallBase *Call) {
- Intrinsic::ID IID = F->getIntrinsicID();
- Type *ReturnType = F->getReturnType();
unsigned BitWidth = ReturnType->getScalarSizeInBits();
switch (IID) {
case Intrinsic::abs:
@@ -6667,8 +6665,6 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1,
break;
}
case Intrinsic::vector_extract: {
- Type *ReturnType = F->getReturnType();
-
// (extract_vector (insert_vector _, X, 0), 0) -> X
unsigned IdxN = cast<ConstantInt>(Op1)->getZExtValue();
Value *X = nullptr;
@@ -6715,7 +6711,7 @@ static Value *simplifyIntrinsic(CallBase *Call, Value *Callee,
return simplifyUnaryIntrinsic(F, Args[0], Q, Call);
if (NumOperands == 2)
- return simplifyBinaryIntrinsic(F, Args[0], Args[1], Q, Call);
+ return simplifyBinaryIntrinsic(IID, F->getReturnType(), Args[0], Args[1], Q, Call);
// Handle intrinsics with 3 or more arguments.
switch (IID) {
>From f3c73933d3ba868d088a9ccff76bb5f4cd4e8446 Mon Sep 17 00:00:00 2001
From: Artem Tyurin <artem.tyurin at gmail.com>
Date: Sun, 11 Feb 2024 12:16:44 +0100
Subject: [PATCH 3/7] Change return type of CreateBinaryIntrinsic from CallInst
to Value
---
llvm/include/llvm/Analysis/ConstantFolding.h | 8 +
.../llvm/Analysis/InstSimplifyFolder.h | 4 +-
llvm/include/llvm/Analysis/TargetFolder.h | 8 +-
llvm/include/llvm/IR/ConstantFold.h | 2 +
llvm/include/llvm/IR/ConstantFolder.h | 19 +-
llvm/include/llvm/IR/IRBuilder.h | 20 +-
llvm/include/llvm/IR/IRBuilderFolder.h | 4 +-
llvm/include/llvm/IR/NoFolder.h | 3 +-
llvm/lib/Analysis/ConstantFolding.cpp | 223 ++++++++++--------
llvm/lib/Analysis/InstructionSimplify.cpp | 8 +-
llvm/lib/IR/ConstantFold.cpp | 26 ++
llvm/lib/IR/IRBuilder.cpp | 11 +-
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 22 +-
.../InstCombine/InstCombineCalls.cpp | 7 +-
.../InstCombine/InstCombineSelect.cpp | 2 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 8 -
llvm/unittests/IR/IRBuilderTest.cpp | 60 ++---
17 files changed, 246 insertions(+), 189 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index 1b194b07e86781..020953248bf8fa 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -22,6 +22,11 @@
#include <stdint.h>
namespace llvm {
+
+namespace Intrinsic {
+typedef unsigned ID;
+}
+
class APInt;
template <typename T> class ArrayRef;
class CallBase;
@@ -186,6 +191,9 @@ Constant *ConstantFoldCall(const CallBase *Call, Function *F,
ArrayRef<Constant *> Operands,
const TargetLibraryInfo *TLI = nullptr);
+Constant *ConstantFoldBinaryIntrinsic(Intrinsic::ID ID, Constant *LHS,
+ Constant *RHS);
+
/// ConstantFoldLoadThroughBitcast - try to cast constant to destination type
/// returning null if unsuccessful. Can cast pointer to pointer or pointer to
/// integer and vice versa if their sizes are equal.
diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
index fbcdf79927d42f..4a6c37654d28ce 100644
--- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h
+++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
@@ -117,8 +117,8 @@ class InstSimplifyFolder final : public IRBuilderFolder {
return simplifyCastInst(Op, V, DestTy, SQ);
}
- Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS,
- Value *RHS) const override {
+ Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
+ Value *RHS) const override {
return simplifyBinaryIntrinsic(ID, LHS->getType(), LHS, RHS, SQ, nullptr);
}
diff --git a/llvm/include/llvm/Analysis/TargetFolder.h b/llvm/include/llvm/Analysis/TargetFolder.h
index cabce12541435d..c9e2e97f0a1862 100644
--- a/llvm/include/llvm/Analysis/TargetFolder.h
+++ b/llvm/include/llvm/Analysis/TargetFolder.h
@@ -191,8 +191,12 @@ class TargetFolder final : public IRBuilderFolder {
return nullptr;
}
- Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS, Value *RHS) const override {
- // TODO: should this be defined?
+ Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
+ Value *RHS) const override {
+ auto *C1 = dyn_cast<Constant>(LHS);
+ auto *C2 = dyn_cast<Constant>(RHS);
+ if (C1 && C2)
+ return ConstantFoldBinaryIntrinsic(ID, C1, C2);
return nullptr;
}
diff --git a/llvm/include/llvm/IR/ConstantFold.h b/llvm/include/llvm/IR/ConstantFold.h
index 77f5f0eb174a2b..b3cad3f8fad8ab 100644
--- a/llvm/include/llvm/IR/ConstantFold.h
+++ b/llvm/include/llvm/IR/ConstantFold.h
@@ -55,6 +55,8 @@ namespace llvm {
Constant *ConstantFoldGetElementPtr(Type *Ty, Constant *C, bool InBounds,
std::optional<unsigned> InRangeIndex,
ArrayRef<Value *> Idxs);
+ Constant *ConstantFoldBinaryIntrinsicInstruction(Intrinsic::ID ID,
+ Constant *V1, Constant *V2);
} // End llvm namespace
#endif
diff --git a/llvm/include/llvm/IR/ConstantFolder.h b/llvm/include/llvm/IR/ConstantFolder.h
index 001d7b1fdf44e8..d24ae874a6792e 100644
--- a/llvm/include/llvm/IR/ConstantFolder.h
+++ b/llvm/include/llvm/IR/ConstantFolder.h
@@ -183,23 +183,12 @@ class ConstantFolder final : public IRBuilderFolder {
return nullptr;
}
- Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS,
- Value *RHS) const override {
+ Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
+ Value *RHS) const override {
auto *LC = dyn_cast<Constant>(LHS);
auto *RC = dyn_cast<Constant>(RHS);
- if (LC && RC) {
- if (ID == Intrinsic::maxnum) {
- return ConstantFP::get(LHS->getType(),
- maxnum(cast<ConstantFP>(LHS)->getValueAPF(),
- cast<ConstantFP>(RHS)->getValueAPF()));
- }
- if (ID == Intrinsic::minnum) {
- return ConstantFP::get(LHS->getType(),
- minnum(cast<ConstantFP>(LHS)->getValueAPF(),
- cast<ConstantFP>(RHS)->getValueAPF()));
- }
- // TODO: use switch, handle more intrinsics
- }
+ if (LC && RC)
+ return ConstantFoldBinaryIntrinsicInstruction(ID, LC, RC);
return nullptr;
}
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index f2922311097e9b..c07ffea7115115 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -962,9 +962,9 @@ class IRBuilderBase {
/// Create a call to intrinsic \p ID with 2 operands which is mangled on the
/// first type.
- CallInst *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
- Instruction *FMFSource = nullptr,
- const Twine &Name = "");
+ Value *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
+ Instruction *FMFSource = nullptr,
+ const Twine &Name = "");
/// Create a call to intrinsic \p ID with \p Args, mangled using \p Types. If
/// \p FMFSource is provided, copy fast-math-flags from that instruction to
@@ -983,7 +983,7 @@ class IRBuilderBase {
const Twine &Name = "");
/// Create call to the minnum intrinsic.
- CallInst *CreateMinNum(Value *LHS, Value *RHS, const Twine &Name = "") {
+ Value *CreateMinNum(Value *LHS, Value *RHS, const Twine &Name = "") {
if (IsFPConstrained) {
return CreateConstrainedFPUnroundedBinOp(
Intrinsic::experimental_constrained_minnum, LHS, RHS, nullptr, Name);
@@ -993,7 +993,7 @@ class IRBuilderBase {
}
/// Create call to the maxnum intrinsic.
- CallInst *CreateMaxNum(Value *LHS, Value *RHS, const Twine &Name = "") {
+ Value *CreateMaxNum(Value *LHS, Value *RHS, const Twine &Name = "") {
if (IsFPConstrained) {
return CreateConstrainedFPUnroundedBinOp(
Intrinsic::experimental_constrained_maxnum, LHS, RHS, nullptr, Name);
@@ -1003,19 +1003,19 @@ class IRBuilderBase {
}
/// Create call to the minimum intrinsic.
- CallInst *CreateMinimum(Value *LHS, Value *RHS, const Twine &Name = "") {
+ Value *CreateMinimum(Value *LHS, Value *RHS, const Twine &Name = "") {
return CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS, nullptr, Name);
}
/// Create call to the maximum intrinsic.
- CallInst *CreateMaximum(Value *LHS, Value *RHS, const Twine &Name = "") {
+ Value *CreateMaximum(Value *LHS, Value *RHS, const Twine &Name = "") {
return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name);
}
/// Create call to the copysign intrinsic.
- CallInst *CreateCopySign(Value *LHS, Value *RHS,
- Instruction *FMFSource = nullptr,
- const Twine &Name = "") {
+ Value *CreateCopySign(Value *LHS, Value *RHS,
+ Instruction *FMFSource = nullptr,
+ const Twine &Name = "") {
return CreateBinaryIntrinsic(Intrinsic::copysign, LHS, RHS, FMFSource,
Name);
}
diff --git a/llvm/include/llvm/IR/IRBuilderFolder.h b/llvm/include/llvm/IR/IRBuilderFolder.h
index 3e0fd093b3f66f..ac9740710d4473 100644
--- a/llvm/include/llvm/IR/IRBuilderFolder.h
+++ b/llvm/include/llvm/IR/IRBuilderFolder.h
@@ -73,8 +73,8 @@ class IRBuilderFolder {
virtual Value *FoldCast(Instruction::CastOps Op, Value *V,
Type *DestTy) const = 0;
- virtual Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS,
- Value *RHS) const = 0;
+ virtual Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
+ Value *RHS) const = 0;
//===--------------------------------------------------------------------===//
// Cast/Conversion Operators
diff --git a/llvm/include/llvm/IR/NoFolder.h b/llvm/include/llvm/IR/NoFolder.h
index 787dd10b4cbaac..49c80f48a01b1d 100644
--- a/llvm/include/llvm/IR/NoFolder.h
+++ b/llvm/include/llvm/IR/NoFolder.h
@@ -112,7 +112,8 @@ class NoFolder final : public IRBuilderFolder {
return nullptr;
}
- Value *FoldBinaryIntrinsics(Intrinsic::ID ID, Value *LHS, Value *RHS) const override {
+ Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
+ Value *RHS) const override {
return nullptr;
}
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 90da3390eab324..a9f7f8229c7ba8 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -2529,33 +2529,75 @@ static Constant *evaluateCompare(const APFloat &Op1, const APFloat &Op2,
return nullptr;
}
-static Constant *ConstantFoldScalarCall2(StringRef Name,
- Intrinsic::ID IntrinsicID,
- Type *Ty,
- ArrayRef<Constant *> Operands,
- const TargetLibraryInfo *TLI,
- const CallBase *Call) {
- assert(Operands.size() == 2 && "Wrong number of operands.");
+static Constant *ConstantFoldLibCall2(StringRef Name, Type *Ty,
+ ArrayRef<Constant *> Operands,
+ const TargetLibraryInfo *TLI) {
+ if (!TLI)
+ return nullptr;
- if (Ty->isFloatingPointTy()) {
- // TODO: We should have undef handling for all of the FP intrinsics that
- // are attempted to be folded in this function.
- bool IsOp0Undef = isa<UndefValue>(Operands[0]);
- bool IsOp1Undef = isa<UndefValue>(Operands[1]);
- switch (IntrinsicID) {
- case Intrinsic::maxnum:
- case Intrinsic::minnum:
- case Intrinsic::maximum:
- case Intrinsic::minimum:
- // If one argument is undef, return the other argument.
- if (IsOp0Undef)
- return Operands[1];
- if (IsOp1Undef)
- return Operands[0];
- break;
+ LibFunc Func = NotLibFunc;
+ if (!TLI->getLibFunc(Name, Func))
+ return nullptr;
+
+ const auto *Op1 = dyn_cast<ConstantFP>(Operands[0]);
+ if (!Op1)
+ return nullptr;
+
+ const auto *Op2 = dyn_cast<ConstantFP>(Operands[1]);
+ if (!Op2)
+ return nullptr;
+
+ const APFloat &Op1V = Op1->getValueAPF();
+ const APFloat &Op2V = Op2->getValueAPF();
+
+ switch (Func) {
+ default:
+ break;
+ case LibFunc_pow:
+ case LibFunc_powf:
+ case LibFunc_pow_finite:
+ case LibFunc_powf_finite:
+ if (TLI->has(Func))
+ return ConstantFoldBinaryFP(pow, Op1V, Op2V, Ty);
+ break;
+ case LibFunc_fmod:
+ case LibFunc_fmodf:
+ if (TLI->has(Func)) {
+ APFloat V = Op1->getValueAPF();
+ if (APFloat::opStatus::opOK == V.mod(Op2->getValueAPF()))
+ return ConstantFP::get(Ty->getContext(), V);
}
+ break;
+ case LibFunc_remainder:
+ case LibFunc_remainderf:
+ if (TLI->has(Func)) {
+ APFloat V = Op1->getValueAPF();
+ if (APFloat::opStatus::opOK == V.remainder(Op2->getValueAPF()))
+ return ConstantFP::get(Ty->getContext(), V);
+ }
+ break;
+ case LibFunc_atan2:
+ case LibFunc_atan2f:
+ // atan2(+/-0.0, +/-0.0) is known to raise an exception on some libm
+ // (Solaris), so we do not assume a known result for that.
+ if (Op1V.isZero() && Op2V.isZero())
+ return nullptr;
+ [[fallthrough]];
+ case LibFunc_atan2_finite:
+ case LibFunc_atan2f_finite:
+ if (TLI->has(Func))
+ return ConstantFoldBinaryFP(atan2, Op1V, Op2V, Ty);
+ break;
}
+ return nullptr;
+}
+
+static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
+ ArrayRef<Constant *> Operands,
+ const CallBase *Call) {
+ assert(Operands.size() == 2 && "Wrong number of operands.");
+
if (const auto *Op1 = dyn_cast<ConstantFP>(Operands[0])) {
const APFloat &Op1V = Op1->getValueAPF();
@@ -2564,36 +2606,38 @@ static Constant *ConstantFoldScalarCall2(StringRef Name,
return nullptr;
const APFloat &Op2V = Op2->getValueAPF();
- if (const auto *ConstrIntr = dyn_cast<ConstrainedFPIntrinsic>(Call)) {
- RoundingMode RM = getEvaluationRoundingMode(ConstrIntr);
- APFloat Res = Op1V;
- APFloat::opStatus St;
- switch (IntrinsicID) {
- default:
+ if (Call) {
+ if (const auto *ConstrIntr = dyn_cast<ConstrainedFPIntrinsic>(Call)) {
+ RoundingMode RM = getEvaluationRoundingMode(ConstrIntr);
+ APFloat Res = Op1V;
+ APFloat::opStatus St;
+ switch (IntrinsicID) {
+ default:
+ return nullptr;
+ case Intrinsic::experimental_constrained_fadd:
+ St = Res.add(Op2V, RM);
+ break;
+ case Intrinsic::experimental_constrained_fsub:
+ St = Res.subtract(Op2V, RM);
+ break;
+ case Intrinsic::experimental_constrained_fmul:
+ St = Res.multiply(Op2V, RM);
+ break;
+ case Intrinsic::experimental_constrained_fdiv:
+ St = Res.divide(Op2V, RM);
+ break;
+ case Intrinsic::experimental_constrained_frem:
+ St = Res.mod(Op2V);
+ break;
+ case Intrinsic::experimental_constrained_fcmp:
+ case Intrinsic::experimental_constrained_fcmps:
+ return evaluateCompare(Op1V, Op2V, ConstrIntr);
+ }
+ if (mayFoldConstrained(
+ const_cast<ConstrainedFPIntrinsic *>(ConstrIntr), St))
+ return ConstantFP::get(Ty->getContext(), Res);
return nullptr;
- case Intrinsic::experimental_constrained_fadd:
- St = Res.add(Op2V, RM);
- break;
- case Intrinsic::experimental_constrained_fsub:
- St = Res.subtract(Op2V, RM);
- break;
- case Intrinsic::experimental_constrained_fmul:
- St = Res.multiply(Op2V, RM);
- break;
- case Intrinsic::experimental_constrained_fdiv:
- St = Res.divide(Op2V, RM);
- break;
- case Intrinsic::experimental_constrained_frem:
- St = Res.mod(Op2V);
- break;
- case Intrinsic::experimental_constrained_fcmp:
- case Intrinsic::experimental_constrained_fcmps:
- return evaluateCompare(Op1V, Op2V, ConstrIntr);
}
- if (mayFoldConstrained(const_cast<ConstrainedFPIntrinsic *>(ConstrIntr),
- St))
- return ConstantFP::get(Ty->getContext(), Res);
- return nullptr;
}
switch (IntrinsicID) {
@@ -2627,52 +2671,6 @@ static Constant *ConstantFoldScalarCall2(StringRef Name,
return ConstantFP::get(Ty->getContext(), Op1V * Op2V);
}
- if (!TLI)
- return nullptr;
-
- LibFunc Func = NotLibFunc;
- if (!TLI->getLibFunc(Name, Func))
- return nullptr;
-
- switch (Func) {
- default:
- break;
- case LibFunc_pow:
- case LibFunc_powf:
- case LibFunc_pow_finite:
- case LibFunc_powf_finite:
- if (TLI->has(Func))
- return ConstantFoldBinaryFP(pow, Op1V, Op2V, Ty);
- break;
- case LibFunc_fmod:
- case LibFunc_fmodf:
- if (TLI->has(Func)) {
- APFloat V = Op1->getValueAPF();
- if (APFloat::opStatus::opOK == V.mod(Op2->getValueAPF()))
- return ConstantFP::get(Ty->getContext(), V);
- }
- break;
- case LibFunc_remainder:
- case LibFunc_remainderf:
- if (TLI->has(Func)) {
- APFloat V = Op1->getValueAPF();
- if (APFloat::opStatus::opOK == V.remainder(Op2->getValueAPF()))
- return ConstantFP::get(Ty->getContext(), V);
- }
- break;
- case LibFunc_atan2:
- case LibFunc_atan2f:
- // atan2(+/-0.0, +/-0.0) is known to raise an exception on some libm
- // (Solaris), so we do not assume a known result for that.
- if (Op1V.isZero() && Op2V.isZero())
- return nullptr;
- [[fallthrough]];
- case LibFunc_atan2_finite:
- case LibFunc_atan2f_finite:
- if (TLI->has(Func))
- return ConstantFoldBinaryFP(atan2, Op1V, Op2V, Ty);
- break;
- }
} else if (auto *Op2C = dyn_cast<ConstantInt>(Operands[1])) {
switch (IntrinsicID) {
case Intrinsic::ldexp: {
@@ -3163,8 +3161,30 @@ static Constant *ConstantFoldScalarCall(StringRef Name,
if (Operands.size() == 1)
return ConstantFoldScalarCall1(Name, IntrinsicID, Ty, Operands, TLI, Call);
- if (Operands.size() == 2)
- return ConstantFoldScalarCall2(Name, IntrinsicID, Ty, Operands, TLI, Call);
+ if (Operands.size() == 2) {
+ if (Ty->isFloatingPointTy()) {
+ // TODO: We should have undef handling for all of the FP intrinsics that
+ // are attempted to be folded in this function.
+ bool IsOp0Undef = isa<UndefValue>(Operands[0]);
+ bool IsOp1Undef = isa<UndefValue>(Operands[1]);
+ switch (IntrinsicID) {
+ case Intrinsic::maxnum:
+ case Intrinsic::minnum:
+ case Intrinsic::maximum:
+ case Intrinsic::minimum:
+ // If one argument is undef, return the other argument.
+ if (IsOp0Undef)
+ return Operands[1];
+ if (IsOp1Undef)
+ return Operands[0];
+ break;
+ }
+ }
+ if (auto *FoldedLibCall = ConstantFoldLibCall2(Name, Ty, Operands, TLI)) {
+ return FoldedLibCall;
+ }
+ return ConstantFoldIntrinsicCall2(IntrinsicID, Ty, Operands, Call);
+ }
if (Operands.size() == 3)
return ConstantFoldScalarCall3(Name, IntrinsicID, Ty, Operands, TLI, Call);
@@ -3371,6 +3391,11 @@ ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
} // end anonymous namespace
+Constant *llvm::ConstantFoldBinaryIntrinsic(Intrinsic::ID ID, Constant *LHS,
+ Constant *RHS) {
+ return ConstantFoldIntrinsicCall2(ID, LHS->getType(), {LHS, RHS}, nullptr);
+}
+
Constant *llvm::ConstantFoldCall(const CallBase *Call, Function *F,
ArrayRef<Constant *> Operands,
const TargetLibraryInfo *TLI) {
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 39fac762df77ae..e73b6d669d56a4 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -6634,19 +6634,21 @@ Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType, Value
// float, if the ninf flag is set.
const APFloat *C;
if (match(Op1, m_APFloat(C)) &&
- (C->isInfinity() || (Call->hasNoInfs() && C->isLargest()))) {
+ (C->isInfinity() || (Call && Call->hasNoInfs() && C->isLargest()))) {
// minnum(X, -inf) -> -inf
// maxnum(X, +inf) -> +inf
// minimum(X, -inf) -> -inf if nnan
// maximum(X, +inf) -> +inf if nnan
- if (C->isNegative() == IsMin && (!PropagateNaN || Call->hasNoNaNs()))
+ if (C->isNegative() == IsMin &&
+ (!PropagateNaN || (Call && Call->hasNoNaNs())))
return ConstantFP::get(ReturnType, *C);
// minnum(X, +inf) -> X if nnan
// maxnum(X, -inf) -> X if nnan
// minimum(X, +inf) -> X
// maximum(X, -inf) -> X
- if (C->isNegative() != IsMin && (PropagateNaN || Call->hasNoNaNs()))
+ if (C->isNegative() != IsMin &&
+ (PropagateNaN || (Call && Call->hasNoNaNs())))
return Op0;
}
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 034e397bc69fce..35a6f966833a3b 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -1735,3 +1735,29 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C,
return nullptr;
}
+
+Constant *llvm::ConstantFoldBinaryIntrinsicInstruction(Intrinsic::ID ID,
+ Constant *LHS,
+ Constant *RHS) {
+ auto *LC = dyn_cast<ConstantFP>(LHS);
+ if (!LC)
+ return nullptr;
+ auto *RC = dyn_cast<ConstantFP>(RHS);
+ if (!RC)
+ return nullptr;
+ auto LVal = LC->getValueAPF();
+ auto RVal = RC->getValueAPF();
+ switch (ID) {
+ case Intrinsic::maxnum:
+ return ConstantFP::get(LHS->getType(), maxnum(LVal, RVal));
+ case Intrinsic::minnum:
+ return ConstantFP::get(LHS->getType(), minnum(LVal, RVal));
+ case Intrinsic::maximum:
+ return ConstantFP::get(LHS->getType(), maximum(LVal, RVal));
+ case Intrinsic::minimum:
+ return ConstantFP::get(LHS->getType(), minimum(LVal, RVal));
+ default:
+ break;
+ }
+ return nullptr;
+}
\ No newline at end of file
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index e5015c1368ceae..c70ce3fd256793 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -918,12 +918,11 @@ CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
return createCallHelper(Fn, {V}, Name, FMFSource);
}
-CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
- Value *RHS,
- Instruction *FMFSource,
- const Twine &Name) {
- if (Value *V = Folder.FoldBinaryIntrinsics(ID, LHS, RHS))
- return (CallInst *) V; // TODO: should return value be changed to Value *?
+Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
+ Value *RHS, Instruction *FMFSource,
+ const Twine &Name) {
+ if (auto *V = Folder.FoldBinaryIntrinsic(ID, LHS, RHS))
+ return V;
Module *M = BB->getModule();
Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index fb829fab0a2c19..6a87e3b892b9d8 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -767,19 +767,25 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
// Checking for NaN before canonicalization provides better fidelity when
// mapping other operations onto fmed3 since the order of operands is
// unchanged.
- CallInst *NewCall = nullptr;
+ Value *V = nullptr;
if (match(Src0, PatternMatch::m_NaN()) || isa<UndefValue>(Src0)) {
- NewCall = IC.Builder.CreateMinNum(Src1, Src2);
+ V = IC.Builder.CreateMinNum(Src1, Src2);
+ // llvm::dbgs() << "1 " << *V << "\n";
} else if (match(Src1, PatternMatch::m_NaN()) || isa<UndefValue>(Src1)) {
- NewCall = IC.Builder.CreateMinNum(Src0, Src2);
+ V = IC.Builder.CreateMinNum(Src0, Src2);
+ // llvm::dbgs() << "2 " << *V << "\n";
} else if (match(Src2, PatternMatch::m_NaN()) || isa<UndefValue>(Src2)) {
- NewCall = IC.Builder.CreateMaxNum(Src0, Src1);
+ V = IC.Builder.CreateMaxNum(Src0, Src1);
+ // llvm::dbgs() << "3 " << *V << "\n";
}
- if (NewCall) {
- NewCall->copyFastMathFlags(&II);
- NewCall->takeName(&II);
- return IC.replaceInstUsesWith(II, NewCall);
+ if (V) {
+ if (auto *CI = dyn_cast<CallInst>(V)) {
+ CI->copyFastMathFlags(&II);
+ CI->takeName(&II);
+ return IC.replaceInstUsesWith(II, CI);
+ }
+ return IC.replaceInstUsesWith(II, V);
}
bool Swap = false;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index ed5d44757fbeb6..773f93737e67e0 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -2257,13 +2257,14 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
default:
llvm_unreachable("unexpected intrinsic ID");
}
- Instruction *NewCall = Builder.CreateBinaryIntrinsic(
+ auto *V = Builder.CreateBinaryIntrinsic(
IID, X, ConstantFP::get(Arg0->getType(), Res), II);
// TODO: Conservatively intersecting FMF. If Res == C2, the transform
// was a simplification (so Arg0 and its original flags could
// propagate?)
- NewCall->andIRFlags(M);
- return replaceInstUsesWith(*II, NewCall);
+ if (auto *CI = dyn_cast<CallInst>(V))
+ CI->andIRFlags(M);
+ return replaceInstUsesWith(*II, V);
}
}
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 527037881edb19..915ed865ab5d38 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1191,7 +1191,7 @@ static Value *canonicalizeSPF(ICmpInst &Cmp, Value *TrueVal, Value *FalseVal,
match(RHS, m_NSWNeg(m_Specific(LHS)));
Constant *IntMinIsPoisonC =
ConstantInt::get(Type::getInt1Ty(Cmp.getContext()), IntMinIsPoison);
- Instruction *Abs =
+ Value *Abs =
IC.Builder.CreateBinaryIntrinsic(Intrinsic::abs, LHS, IntMinIsPoisonC);
if (SPF == SelectPatternFlavor::SPF_NABS)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3e95df4b8c33e6..efbea5d4dbaaf0 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14120,16 +14120,8 @@ class HorizontalReduction {
case RecurKind::FMin:
return Builder.CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS);
case RecurKind::FMaximum:
- if (IsConstant)
- return ConstantFP::get(LHS->getType(),
- maximum(cast<ConstantFP>(LHS)->getValueAPF(),
- cast<ConstantFP>(RHS)->getValueAPF()));
return Builder.CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS);
case RecurKind::FMinimum:
- if (IsConstant)
- return ConstantFP::get(LHS->getType(),
- minimum(cast<ConstantFP>(LHS)->getValueAPF(),
- cast<ConstantFP>(RHS)->getValueAPF()));
return Builder.CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS);
case RecurKind::SMax:
if (IsConstant || UseSelect) {
diff --git a/llvm/unittests/IR/IRBuilderTest.cpp b/llvm/unittests/IR/IRBuilderTest.cpp
index d15ff9dd51a4c4..74ef72f42d9057 100644
--- a/llvm/unittests/IR/IRBuilderTest.cpp
+++ b/llvm/unittests/IR/IRBuilderTest.cpp
@@ -57,7 +57,7 @@ TEST_F(IRBuilderTest, Intrinsics) {
IRBuilder<> Builder(BB);
Value *V;
Instruction *I;
- CallInst *Call;
+ Value *Result;
IntrinsicInst *II;
V = Builder.CreateLoad(GV->getValueType(), GV);
@@ -65,78 +65,80 @@ TEST_F(IRBuilderTest, Intrinsics) {
I->setHasNoInfs(true);
I->setHasNoNaNs(false);
- Call = Builder.CreateMinNum(V, V);
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateMinNum(V, V);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::minnum);
- Call = Builder.CreateMaxNum(V, V);
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateMaxNum(V, V);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::maxnum);
- Call = Builder.CreateMinimum(V, V);
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateMinimum(V, V);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::minimum);
- Call = Builder.CreateMaximum(V, V);
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateMaximum(V, V);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::maximum);
- Call = Builder.CreateIntrinsic(Intrinsic::readcyclecounter, {}, {});
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateIntrinsic(Intrinsic::readcyclecounter, {}, {});
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::readcyclecounter);
- Call = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, V);
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, V);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::fabs);
EXPECT_FALSE(II->hasNoInfs());
EXPECT_FALSE(II->hasNoNaNs());
- Call = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, V, I);
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, V, I);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::fabs);
EXPECT_TRUE(II->hasNoInfs());
EXPECT_FALSE(II->hasNoNaNs());
- Call = Builder.CreateBinaryIntrinsic(Intrinsic::pow, V, V);
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateBinaryIntrinsic(Intrinsic::pow, V, V);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::pow);
EXPECT_FALSE(II->hasNoInfs());
EXPECT_FALSE(II->hasNoNaNs());
- Call = Builder.CreateBinaryIntrinsic(Intrinsic::pow, V, V, I);
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateBinaryIntrinsic(Intrinsic::pow, V, V, I);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::pow);
EXPECT_TRUE(II->hasNoInfs());
EXPECT_FALSE(II->hasNoNaNs());
- Call = Builder.CreateIntrinsic(Intrinsic::fma, {V->getType()}, {V, V, V});
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateIntrinsic(Intrinsic::fma, {V->getType()}, {V, V, V});
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::fma);
EXPECT_FALSE(II->hasNoInfs());
EXPECT_FALSE(II->hasNoNaNs());
- Call = Builder.CreateIntrinsic(Intrinsic::fma, {V->getType()}, {V, V, V}, I);
- II = cast<IntrinsicInst>(Call);
+ Result =
+ Builder.CreateIntrinsic(Intrinsic::fma, {V->getType()}, {V, V, V}, I);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::fma);
EXPECT_TRUE(II->hasNoInfs());
EXPECT_FALSE(II->hasNoNaNs());
- Call = Builder.CreateIntrinsic(Intrinsic::fma, {V->getType()}, {V, V, V}, I);
- II = cast<IntrinsicInst>(Call);
+ Result =
+ Builder.CreateIntrinsic(Intrinsic::fma, {V->getType()}, {V, V, V}, I);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::fma);
EXPECT_TRUE(II->hasNoInfs());
EXPECT_FALSE(II->hasNoNaNs());
- Call = Builder.CreateUnaryIntrinsic(Intrinsic::roundeven, V);
- II = cast<IntrinsicInst>(Call);
+ Result = Builder.CreateUnaryIntrinsic(Intrinsic::roundeven, V);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::roundeven);
EXPECT_FALSE(II->hasNoInfs());
EXPECT_FALSE(II->hasNoNaNs());
- Call = Builder.CreateIntrinsic(
+ Result = Builder.CreateIntrinsic(
Intrinsic::set_rounding, {},
{Builder.getInt32(static_cast<uint32_t>(RoundingMode::TowardZero))});
- II = cast<IntrinsicInst>(Call);
+ II = cast<IntrinsicInst>(Result);
EXPECT_EQ(II->getIntrinsicID(), Intrinsic::set_rounding);
}
>From a6423b6c83be7c641dda446a058a3fe8e38780fe Mon Sep 17 00:00:00 2001
From: Artem Tyurin <artem.tyurin at gmail.com>
Date: Sun, 11 Feb 2024 12:41:24 +0100
Subject: [PATCH 4/7] Code formatting
---
.../llvm/Analysis/InstructionSimplify.h | 6 +--
llvm/lib/Analysis/ConstantFolding.cpp | 37 ++++++++++---------
llvm/lib/Analysis/InstructionSimplify.cpp | 10 +++--
3 files changed, 28 insertions(+), 25 deletions(-)
diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h
index c48812c7923d65..03d7ad12c12d8f 100644
--- a/llvm/include/llvm/Analysis/InstructionSimplify.h
+++ b/llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -187,9 +187,9 @@ Value *simplifyCastInst(unsigned CastOpc, Value *Op, Type *Ty,
const SimplifyQuery &Q);
/// Given operands for a BinaryIntrinsic, fold the result or return null.
-Value *simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType, Value *Op0, Value *Op1,
- const SimplifyQuery &Q,
- const CallBase *Call);
+Value *simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType, Value *Op0,
+ Value *Op1, const SimplifyQuery &Q,
+ const CallBase *Call);
/// Given operands for a ShuffleVectorInst, fold the result or return null.
/// See class ShuffleVectorInst for a description of the mask representation.
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index a9f7f8229c7ba8..10cebc114f1fd1 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -2598,6 +2598,25 @@ static Constant *ConstantFoldIntrinsicCall2(Intrinsic::ID IntrinsicID, Type *Ty,
const CallBase *Call) {
assert(Operands.size() == 2 && "Wrong number of operands.");
+ if (Ty->isFloatingPointTy()) {
+ // TODO: We should have undef handling for all of the FP intrinsics that
+ // are attempted to be folded in this function.
+ bool IsOp0Undef = isa<UndefValue>(Operands[0]);
+ bool IsOp1Undef = isa<UndefValue>(Operands[1]);
+ switch (IntrinsicID) {
+ case Intrinsic::maxnum:
+ case Intrinsic::minnum:
+ case Intrinsic::maximum:
+ case Intrinsic::minimum:
+ // If one argument is undef, return the other argument.
+ if (IsOp0Undef)
+ return Operands[1];
+ if (IsOp1Undef)
+ return Operands[0];
+ break;
+ }
+ }
+
if (const auto *Op1 = dyn_cast<ConstantFP>(Operands[0])) {
const APFloat &Op1V = Op1->getValueAPF();
@@ -3162,24 +3181,6 @@ static Constant *ConstantFoldScalarCall(StringRef Name,
return ConstantFoldScalarCall1(Name, IntrinsicID, Ty, Operands, TLI, Call);
if (Operands.size() == 2) {
- if (Ty->isFloatingPointTy()) {
- // TODO: We should have undef handling for all of the FP intrinsics that
- // are attempted to be folded in this function.
- bool IsOp0Undef = isa<UndefValue>(Operands[0]);
- bool IsOp1Undef = isa<UndefValue>(Operands[1]);
- switch (IntrinsicID) {
- case Intrinsic::maxnum:
- case Intrinsic::minnum:
- case Intrinsic::maximum:
- case Intrinsic::minimum:
- // If one argument is undef, return the other argument.
- if (IsOp0Undef)
- return Operands[1];
- if (IsOp1Undef)
- return Operands[0];
- break;
- }
- }
if (auto *FoldedLibCall = ConstantFoldLibCall2(Name, Ty, Operands, TLI)) {
return FoldedLibCall;
}
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index e73b6d669d56a4..51f390db424cdb 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -6374,9 +6374,10 @@ static Value *foldMinimumMaximumSharedOp(Intrinsic::ID IID, Value *Op0,
return nullptr;
}
-Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType, Value *Op0, Value *Op1,
- const SimplifyQuery &Q,
- const CallBase *Call) {
+Value *llvm::simplifyBinaryIntrinsic(Intrinsic::ID IID, Type *ReturnType,
+ Value *Op0, Value *Op1,
+ const SimplifyQuery &Q,
+ const CallBase *Call) {
unsigned BitWidth = ReturnType->getScalarSizeInBits();
switch (IID) {
case Intrinsic::abs:
@@ -6708,7 +6709,8 @@ static Value *simplifyIntrinsic(CallBase *Call, Value *Callee,
return simplifyUnaryIntrinsic(F, Args[0], Q, Call);
if (NumOperands == 2)
- return simplifyBinaryIntrinsic(IID, F->getReturnType(), Args[0], Args[1], Q, Call);
+ return simplifyBinaryIntrinsic(IID, F->getReturnType(), Args[0], Args[1], Q,
+ Call);
// Handle intrinsics with 3 or more arguments.
switch (IID) {
>From 405029085e71ae84dcc334c4b6d89c13973d0130 Mon Sep 17 00:00:00 2001
From: Artem Tyurin <artem.tyurin at gmail.com>
Date: Thu, 22 Feb 2024 14:11:31 +0100
Subject: [PATCH 5/7] Add type parameter to FoldBinaryIntrinsic
---
llvm/include/llvm/Analysis/ConstantFolding.h | 4 ++--
llvm/include/llvm/Analysis/InstSimplifyFolder.h | 6 +++---
llvm/include/llvm/Analysis/TargetFolder.h | 6 +++---
llvm/include/llvm/IR/ConstantFold.h | 3 ++-
llvm/include/llvm/IR/ConstantFolder.h | 6 +++---
llvm/include/llvm/IR/IRBuilderFolder.h | 4 ++--
llvm/include/llvm/IR/NoFolder.h | 4 ++--
llvm/lib/Analysis/ConstantFolding.cpp | 4 ++--
llvm/lib/IR/ConstantFold.cpp | 13 +++++++------
llvm/lib/IR/IRBuilder.cpp | 4 ++--
.../Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 4 ----
11 files changed, 28 insertions(+), 30 deletions(-)
diff --git a/llvm/include/llvm/Analysis/ConstantFolding.h b/llvm/include/llvm/Analysis/ConstantFolding.h
index 020953248bf8fa..d860d7a3d37e63 100644
--- a/llvm/include/llvm/Analysis/ConstantFolding.h
+++ b/llvm/include/llvm/Analysis/ConstantFolding.h
@@ -24,7 +24,7 @@
namespace llvm {
namespace Intrinsic {
-typedef unsigned ID;
+using ID = unsigned;
}
class APInt;
@@ -192,7 +192,7 @@ Constant *ConstantFoldCall(const CallBase *Call, Function *F,
const TargetLibraryInfo *TLI = nullptr);
Constant *ConstantFoldBinaryIntrinsic(Intrinsic::ID ID, Constant *LHS,
- Constant *RHS);
+ Constant *RHS, Type *Ty);
/// ConstantFoldLoadThroughBitcast - try to cast constant to destination type
/// returning null if unsuccessful. Can cast pointer to pointer or pointer to
diff --git a/llvm/include/llvm/Analysis/InstSimplifyFolder.h b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
index 4a6c37654d28ce..298fd05269e5ce 100644
--- a/llvm/include/llvm/Analysis/InstSimplifyFolder.h
+++ b/llvm/include/llvm/Analysis/InstSimplifyFolder.h
@@ -117,9 +117,9 @@ class InstSimplifyFolder final : public IRBuilderFolder {
return simplifyCastInst(Op, V, DestTy, SQ);
}
- Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
- Value *RHS) const override {
- return simplifyBinaryIntrinsic(ID, LHS->getType(), LHS, RHS, SQ, nullptr);
+ Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
+ Type *Ty) const override {
+ return simplifyBinaryIntrinsic(ID, Ty, LHS, RHS, SQ, nullptr);
}
//===--------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/Analysis/TargetFolder.h b/llvm/include/llvm/Analysis/TargetFolder.h
index c9e2e97f0a1862..bbd9d24dfa2ae6 100644
--- a/llvm/include/llvm/Analysis/TargetFolder.h
+++ b/llvm/include/llvm/Analysis/TargetFolder.h
@@ -191,12 +191,12 @@ class TargetFolder final : public IRBuilderFolder {
return nullptr;
}
- Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
- Value *RHS) const override {
+ Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
+ Type *Ty) const override {
auto *C1 = dyn_cast<Constant>(LHS);
auto *C2 = dyn_cast<Constant>(RHS);
if (C1 && C2)
- return ConstantFoldBinaryIntrinsic(ID, C1, C2);
+ return ConstantFoldBinaryIntrinsic(ID, C1, C2, Ty);
return nullptr;
}
diff --git a/llvm/include/llvm/IR/ConstantFold.h b/llvm/include/llvm/IR/ConstantFold.h
index b3cad3f8fad8ab..567106a6d13b93 100644
--- a/llvm/include/llvm/IR/ConstantFold.h
+++ b/llvm/include/llvm/IR/ConstantFold.h
@@ -56,7 +56,8 @@ namespace llvm {
std::optional<unsigned> InRangeIndex,
ArrayRef<Value *> Idxs);
Constant *ConstantFoldBinaryIntrinsicInstruction(Intrinsic::ID ID,
- Constant *V1, Constant *V2);
+ Constant *V1, Constant *V2,
+ Type *Ty);
} // End llvm namespace
#endif
diff --git a/llvm/include/llvm/IR/ConstantFolder.h b/llvm/include/llvm/IR/ConstantFolder.h
index d24ae874a6792e..52b3e8411f05d5 100644
--- a/llvm/include/llvm/IR/ConstantFolder.h
+++ b/llvm/include/llvm/IR/ConstantFolder.h
@@ -183,12 +183,12 @@ class ConstantFolder final : public IRBuilderFolder {
return nullptr;
}
- Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
- Value *RHS) const override {
+ Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
+ Type *Ty) const override {
auto *LC = dyn_cast<Constant>(LHS);
auto *RC = dyn_cast<Constant>(RHS);
if (LC && RC)
- return ConstantFoldBinaryIntrinsicInstruction(ID, LC, RC);
+ return ConstantFoldBinaryIntrinsicInstruction(ID, LC, RC, Ty);
return nullptr;
}
diff --git a/llvm/include/llvm/IR/IRBuilderFolder.h b/llvm/include/llvm/IR/IRBuilderFolder.h
index ac9740710d4473..4aea239d439017 100644
--- a/llvm/include/llvm/IR/IRBuilderFolder.h
+++ b/llvm/include/llvm/IR/IRBuilderFolder.h
@@ -73,8 +73,8 @@ class IRBuilderFolder {
virtual Value *FoldCast(Instruction::CastOps Op, Value *V,
Type *DestTy) const = 0;
- virtual Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
- Value *RHS) const = 0;
+ virtual Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
+ Type *Ty) const = 0;
//===--------------------------------------------------------------------===//
// Cast/Conversion Operators
diff --git a/llvm/include/llvm/IR/NoFolder.h b/llvm/include/llvm/IR/NoFolder.h
index 49c80f48a01b1d..0bcbd855340dfa 100644
--- a/llvm/include/llvm/IR/NoFolder.h
+++ b/llvm/include/llvm/IR/NoFolder.h
@@ -112,8 +112,8 @@ class NoFolder final : public IRBuilderFolder {
return nullptr;
}
- Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
- Value *RHS) const override {
+ Value *FoldBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS,
+ Type *Ty) const override {
return nullptr;
}
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 10cebc114f1fd1..2deca3b55fd5ce 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -3393,8 +3393,8 @@ ConstantFoldStructCall(StringRef Name, Intrinsic::ID IntrinsicID,
} // end anonymous namespace
Constant *llvm::ConstantFoldBinaryIntrinsic(Intrinsic::ID ID, Constant *LHS,
- Constant *RHS) {
- return ConstantFoldIntrinsicCall2(ID, LHS->getType(), {LHS, RHS}, nullptr);
+ Constant *RHS, Type *Ty) {
+ return ConstantFoldIntrinsicCall2(ID, Ty, {LHS, RHS}, nullptr);
}
Constant *llvm::ConstantFoldCall(const CallBase *Call, Function *F,
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 35a6f966833a3b..3c8815e86d6c41 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -1738,7 +1738,8 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C,
Constant *llvm::ConstantFoldBinaryIntrinsicInstruction(Intrinsic::ID ID,
Constant *LHS,
- Constant *RHS) {
+ Constant *RHS,
+ Type *Ty) {
auto *LC = dyn_cast<ConstantFP>(LHS);
if (!LC)
return nullptr;
@@ -1749,15 +1750,15 @@ Constant *llvm::ConstantFoldBinaryIntrinsicInstruction(Intrinsic::ID ID,
auto RVal = RC->getValueAPF();
switch (ID) {
case Intrinsic::maxnum:
- return ConstantFP::get(LHS->getType(), maxnum(LVal, RVal));
+ return ConstantFP::get(Ty, maxnum(LVal, RVal));
case Intrinsic::minnum:
- return ConstantFP::get(LHS->getType(), minnum(LVal, RVal));
+ return ConstantFP::get(Ty, minnum(LVal, RVal));
case Intrinsic::maximum:
- return ConstantFP::get(LHS->getType(), maximum(LVal, RVal));
+ return ConstantFP::get(Ty, maximum(LVal, RVal));
case Intrinsic::minimum:
- return ConstantFP::get(LHS->getType(), minimum(LVal, RVal));
+ return ConstantFP::get(Ty, minimum(LVal, RVal));
default:
break;
}
return nullptr;
-}
\ No newline at end of file
+}
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index c70ce3fd256793..0ba2a79b6d58d2 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -921,10 +921,10 @@ CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
Value *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
Value *RHS, Instruction *FMFSource,
const Twine &Name) {
- if (auto *V = Folder.FoldBinaryIntrinsic(ID, LHS, RHS))
- return V;
Module *M = BB->getModule();
Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
+ if (auto *V = Folder.FoldBinaryIntrinsic(ID, LHS, RHS, Fn->getReturnType()))
+ return V;
return createCallHelper(Fn, {LHS, RHS}, Name, FMFSource);
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index 6a87e3b892b9d8..5b7fa13f2e835f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -770,20 +770,16 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Value *V = nullptr;
if (match(Src0, PatternMatch::m_NaN()) || isa<UndefValue>(Src0)) {
V = IC.Builder.CreateMinNum(Src1, Src2);
- // llvm::dbgs() << "1 " << *V << "\n";
} else if (match(Src1, PatternMatch::m_NaN()) || isa<UndefValue>(Src1)) {
V = IC.Builder.CreateMinNum(Src0, Src2);
- // llvm::dbgs() << "2 " << *V << "\n";
} else if (match(Src2, PatternMatch::m_NaN()) || isa<UndefValue>(Src2)) {
V = IC.Builder.CreateMaxNum(Src0, Src1);
- // llvm::dbgs() << "3 " << *V << "\n";
}
if (V) {
if (auto *CI = dyn_cast<CallInst>(V)) {
CI->copyFastMathFlags(&II);
CI->takeName(&II);
- return IC.replaceInstUsesWith(II, CI);
}
return IC.replaceInstUsesWith(II, V);
}
>From bb10cd8c082dfb539d2cc1018d88bb702014e30a Mon Sep 17 00:00:00 2001
From: Artem Tyurin <artem.tyurin at gmail.com>
Date: Thu, 22 Feb 2024 14:22:02 +0100
Subject: [PATCH 6/7] Explicit type annotation
---
llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 773f93737e67e0..de19872fdd792d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -2257,7 +2257,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
default:
llvm_unreachable("unexpected intrinsic ID");
}
- auto *V = Builder.CreateBinaryIntrinsic(
+ Value *V = Builder.CreateBinaryIntrinsic(
IID, X, ConstantFP::get(Arg0->getType(), Res), II);
// TODO: Conservatively intersecting FMF. If Res == C2, the transform
// was a simplification (so Arg0 and its original flags could
>From dba29b3ffbe31c9b8195241e4000a08b8e917fd5 Mon Sep 17 00:00:00 2001
From: Artem Tyurin <artem.tyurin at gmail.com>
Date: Thu, 22 Feb 2024 14:49:04 +0100
Subject: [PATCH 7/7] Merge intrinsic cases in SLPVectorizer
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index efbea5d4dbaaf0..ed4398535ddb80 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14116,13 +14116,11 @@ class HorizontalReduction {
return Builder.CreateBinOp((Instruction::BinaryOps)RdxOpcode, LHS, RHS,
Name);
case RecurKind::FMax:
- return Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS);
case RecurKind::FMin:
- return Builder.CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS);
case RecurKind::FMaximum:
- return Builder.CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS);
case RecurKind::FMinimum:
- return Builder.CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS);
+ return Builder.CreateBinaryIntrinsic(getMinMaxReductionIntrinsicOp(Kind),
+ LHS, RHS);
case RecurKind::SMax:
if (IsConstant || UseSelect) {
Value *Cmp = Builder.CreateICmpSGT(LHS, RHS, Name);
More information about the llvm-commits
mailing list