[llvm] Port Swift's merge function pass to llvm: merging functions that differ in constants (PR #68235)
Manman Ren via llvm-commits
llvm-commits at lists.llvm.org
Wed Oct 4 10:10:58 PDT 2023
https://github.com/manman-ren created https://github.com/llvm/llvm-project/pull/68235
See RFC for details: https://discourse.llvm.org/t/rfc-for-moving-swift-s-merge-function-pass-to-llvm/73778
We will need to refactor extension to FunctionComparator/FunctionHash to StructuralHash. This patch adds a new pass which is ported from Swift, and will need to discuss on how to migrate Swift’s pass over after we land this in llvm.
Create this PR to get some early review on the patch.
>From 7e422cecbd5fc28fb0ec699b702d6bccf321f93f Mon Sep 17 00:00:00 2001
From: Manman Ren <mren at fb.com>
Date: Mon, 2 Oct 2023 11:16:58 -0700
Subject: [PATCH] Preliminary patch for merging functions that differ in
constants
---
.../IPO/MergeFunctionsIgnoringConst.h | 34 +
.../Transforms/Utils/FunctionComparator.h | 1 +
.../Utils/FunctionComparatorIgnoringConst.h | 58 +
.../Utils/FunctionHashIgnoringConst.h | 79 +
.../Utils/MergeFunctionsIgnoringConst.h | 29 +
llvm/lib/Passes/PassBuilder.cpp | 1 +
llvm/lib/Passes/PassBuilderPipelines.cpp | 11 +
llvm/lib/Passes/PassRegistry.def | 1 +
llvm/lib/Transforms/IPO/CMakeLists.txt | 1 +
.../IPO/MergeFunctionsIgnoringConst.cpp | 1430 +++++++++++++++++
llvm/lib/Transforms/Utils/CMakeLists.txt | 2 +
.../Utils/FunctionComparatorIgnoringConst.cpp | 107 ++
.../Utils/FunctionHashIgnoringConst.cpp | 620 +++++++
.../unittests/Transforms/Utils/CMakeLists.txt | 1 +
.../Utils/FunctionHashIgnoringConstTest.cpp | 120 ++
15 files changed, 2495 insertions(+)
create mode 100644 llvm/include/llvm/Transforms/IPO/MergeFunctionsIgnoringConst.h
create mode 100644 llvm/include/llvm/Transforms/Utils/FunctionComparatorIgnoringConst.h
create mode 100644 llvm/include/llvm/Transforms/Utils/FunctionHashIgnoringConst.h
create mode 100644 llvm/include/llvm/Transforms/Utils/MergeFunctionsIgnoringConst.h
create mode 100644 llvm/lib/Transforms/IPO/MergeFunctionsIgnoringConst.cpp
create mode 100644 llvm/lib/Transforms/Utils/FunctionComparatorIgnoringConst.cpp
create mode 100644 llvm/lib/Transforms/Utils/FunctionHashIgnoringConst.cpp
create mode 100644 llvm/unittests/Transforms/Utils/FunctionHashIgnoringConstTest.cpp
diff --git a/llvm/include/llvm/Transforms/IPO/MergeFunctionsIgnoringConst.h b/llvm/include/llvm/Transforms/IPO/MergeFunctionsIgnoringConst.h
new file mode 100644
index 000000000000000..f9d55cc40873adc
--- /dev/null
+++ b/llvm/include/llvm/Transforms/IPO/MergeFunctionsIgnoringConst.h
@@ -0,0 +1,34 @@
+//===- MergeFunctionsIgnoringConst.h - Merge Functions ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass transforms simple global variables that never have their address
+// taken. If obviously true, it marks read/write globals as constant, deletes
+// variables only stored to, etc.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_IPO_MERGEFUNCTIONSIGNORINGCONST_H
+#define LLVM_TRANSFORMS_IPO_MERGEFUNCTIONSIGNORINGCONST_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class Module;
+
+/// Merge functions that differ by constants.
+class MergeFuncIgnoringConstPass
+ : public PassInfoMixin<MergeFuncIgnoringConstPass> {
+public:
+ MergeFuncIgnoringConstPass() {}
+ PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_IPO_MERGEFUNCTIONSIGNORINGCONST_H
diff --git a/llvm/include/llvm/Transforms/Utils/FunctionComparator.h b/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
index c28f868039a1f7b..1a314b481c72c61 100644
--- a/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
+++ b/llvm/include/llvm/Transforms/Utils/FunctionComparator.h
@@ -379,6 +379,7 @@ class FunctionComparator {
/// But, we are still not able to compare operands of PHI nodes, since those
/// could be operands from further BBs we didn't scan yet.
/// So it's impossible to use dominance properties in general.
+protected:
mutable DenseMap<const Value*, int> sn_mapL, sn_mapR;
// The global state we will use
diff --git a/llvm/include/llvm/Transforms/Utils/FunctionComparatorIgnoringConst.h b/llvm/include/llvm/Transforms/Utils/FunctionComparatorIgnoringConst.h
new file mode 100644
index 000000000000000..a61e02fa41db762
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/FunctionComparatorIgnoringConst.h
@@ -0,0 +1,58 @@
+//===- FunctionComparatorIgnoringConst.h - Function Comparator --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the FunctionComparatorIgnoringConst class which is used by
+// the MergeFuncIgnoringConst pass for comparing functions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_FUNCTIONCOMPARATORIGNORINGCONST_H
+#define LLVM_TRANSFORMS_UTILS_FUNCTIONCOMPARATORIGNORINGCONST_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/ValueMap.h"
+#include "llvm/Support/AtomicOrdering.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Transforms/Utils/FunctionComparator.h"
+#include <set>
+
+namespace llvm {
+
+/// FunctionComparatorIgnoringConst - Compares two functions to determine
+/// whether or not they will generate machine code with the same behavior.
+class FunctionComparatorIgnoringConst : public FunctionComparator {
+public:
+ FunctionComparatorIgnoringConst(const Function *F1, const Function *F2,
+ GlobalNumberState *GN)
+ : FunctionComparator(F1, F2, GN) {}
+
+ int cmpOperandsIgnoringConsts(const Instruction *L, const Instruction *R,
+ unsigned opIdx);
+
+ int cmpBasicBlocksIgnoringConsts(
+ const BasicBlock *BBL, const BasicBlock *BBR,
+ const std::set<std::pair<int, int>> *InstOpndIndex = nullptr);
+
+ int compareIgnoringConsts(
+ const std::set<std::pair<int, int>> *InstOpndIndex = nullptr);
+
+ int compareConstants(const Constant *L, const Constant *R) const {
+ return cmpConstants(L, R);
+ }
+
+private:
+ // Scratch index for instruction in order during cmpOperandsIgnoringConsts.
+ int index = 0;
+};
+
+} // end namespace llvm
+#endif // LLVM_TRANSFORMS_UTILS_FUNCTIONCOMPARATORIGNORINGCONST_H
diff --git a/llvm/include/llvm/Transforms/Utils/FunctionHashIgnoringConst.h b/llvm/include/llvm/Transforms/Utils/FunctionHashIgnoringConst.h
new file mode 100644
index 000000000000000..d696ae8c2381128
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/FunctionHashIgnoringConst.h
@@ -0,0 +1,79 @@
+//===- FunctionHashIgnoringConst.h - Function Hash -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the FunctionHashIgnoringConst class which is used by the
+// global merge functions that can differ by Constants. This provides stable
+// functions hash that ignores Constants. As for Constants that are ignored,
+// this also track their locations (instruction, operand) indices.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_FUNCTIONHASHIGNORINGCONST_H
+#define LLVM_TRANSFORMS_UTILS_FUNCTIONHASHIGNORINGCONST_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/ValueMap.h"
+#include "llvm/Support/AtomicOrdering.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Transforms/Utils/FunctionComparatorIgnoringConst.h"
+#include <map>
+
+namespace llvm {
+
+class FunctionHashIgnoringConst : public FunctionComparatorIgnoringConst {
+public:
+ using FunctionHash = uint64_t;
+
+ /// Get function hash by ignoring Constant operands.
+ /// This is different than FunctionComparator::functionHash which just hashes
+ /// the opcode. Optionally IdxToConstHash map is passed to return the pair of
+ /// instruction and operand indices to the const hash. Optionally IdxToInst
+ /// map is passed to return a map of instruction index to instruction.
+ static FunctionHash functionHash(
+ Function &F, std::map<int, Instruction *> *IdxToInst = nullptr,
+ std::map<std::pair<int, int>, uint64_t> *IdxToConstHash = nullptr);
+
+private:
+ FunctionHashIgnoringConst(const Function *F1, GlobalNumberState *GN)
+ : FunctionComparatorIgnoringConst(F1, F1, GN) {}
+
+ FunctionHash
+ hashIgnoringConsts(std::map<int, Instruction *> &IdxToInst,
+ std::map<std::pair<int, int>, uint64_t> &IdxToConstHash);
+
+ FunctionHash hashBasicBlocksIgnoringConsts(
+ const BasicBlock *BBL, std::map<int, Instruction *> &IdxToInst,
+ std::map<std::pair<int, int>, uint64_t> &IdxToConstHash);
+
+ FunctionHash hashType(Type *TyL) const;
+ FunctionHash hashValue(const Value *v) const;
+ FunctionHash hashOperation(const Instruction *i,
+ bool &needToCmpOperands) const;
+ FunctionHash hashAttrs(const AttributeList L) const;
+ FunctionHash hashSignature() const;
+
+ FunctionHash hashInlineAsm(const InlineAsm *L) const;
+ FunctionHash hashConstant(const Constant *L) const;
+ FunctionHash hashAPInt(const APInt &L) const;
+ FunctionHash hashAPFloat(const APFloat &L) const;
+ FunctionHash hashGlobalValue(const GlobalValue *L) const;
+ FunctionHash hashGEP(const GEPOperator *GEPL) const;
+ FunctionHash hashOperandBundlesSchema(const CallBase &LCS) const;
+ FunctionHash hashRangeMetadata(const MDNode *L) const;
+
+private:
+ // Scratch index for instruction in order during hashIgnoringConsts.
+ int index = 0;
+};
+
+} // end namespace llvm
+#endif // LLVM_TRANSFORMS_UTILS_FUNCTIONHASHIGNORINGCONST_H
diff --git a/llvm/include/llvm/Transforms/Utils/MergeFunctionsIgnoringConst.h b/llvm/include/llvm/Transforms/Utils/MergeFunctionsIgnoringConst.h
new file mode 100644
index 000000000000000..e63afbb6bbf1718
--- /dev/null
+++ b/llvm/include/llvm/Transforms/Utils/MergeFunctionsIgnoringConst.h
@@ -0,0 +1,29 @@
+//===- MergeFunctionsIgnoringConst.h - Merge Functions ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines helpers used in the MergeFunctionsIgnoringConst.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_MERGEFUNCTIONSIGNORINGCONST_H
+#define LLVM_TRANSFORMS_UTILS_MERGEFUNCTIONSIGNORINGCONST_H
+
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Operator.h"
+
+using namespace llvm;
+
+bool isEligibleInstrunctionForConstantSharing(const Instruction *I);
+
+bool isEligibleOperandForConstantSharing(const Instruction *I, unsigned OpIdx);
+
+bool isEligibleFunction(Function *F);
+
+Value *createCast(IRBuilder<> &Builder, Value *V, Type *DestTy);
+#endif // LLVM_TRANSFORMS_UTILS_MERGEFUNCTIONSIGNORINGCONST_H
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 985ff88139323c6..14a0b62cb9a81c9 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -120,6 +120,7 @@
#include "llvm/Transforms/IPO/LowerTypeTests.h"
#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
#include "llvm/Transforms/IPO/MergeFunctions.h"
+#include "llvm/Transforms/IPO/MergeFunctionsIgnoringConst.h"
#include "llvm/Transforms/IPO/ModuleInliner.h"
#include "llvm/Transforms/IPO/OpenMPOpt.h"
#include "llvm/Transforms/IPO/PartialInlining.h"
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index 78e0e6353056343..4a8051405f67025 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -59,6 +59,7 @@
#include "llvm/Transforms/IPO/LowerTypeTests.h"
#include "llvm/Transforms/IPO/MemProfContextDisambiguation.h"
#include "llvm/Transforms/IPO/MergeFunctions.h"
+#include "llvm/Transforms/IPO/MergeFunctionsIgnoringConst.h"
#include "llvm/Transforms/IPO/ModuleInliner.h"
#include "llvm/Transforms/IPO/OpenMPOpt.h"
#include "llvm/Transforms/IPO/PartialInlining.h"
@@ -175,6 +176,10 @@ static cl::opt<bool> EnableMergeFunctions(
"enable-merge-functions", cl::init(false), cl::Hidden,
cl::desc("Enable function merging as part of the optimization pipeline"));
+static cl::opt<bool> EnableMergeFuncIgnoringConst(
+ "enable-merge-func-ignoring-const", cl::init(false), cl::Hidden,
+ cl::desc("Enable function merger that ignores constants"));
+
static cl::opt<bool> EnablePostPGOLoopRotation(
"enable-post-pgo-loop-rotation", cl::init(true), cl::Hidden,
cl::desc("Run the loop rotation transformation after PGO instrumentation"));
@@ -1628,6 +1633,9 @@ ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
MPM.addPass(buildModuleOptimizationPipeline(
Level, ThinOrFullLTOPhase::ThinLTOPostLink));
+ if (EnableMergeFuncIgnoringConst)
+ MPM.addPass(MergeFuncIgnoringConstPass());
+
// Emit annotation remarks.
addAnnotationRemarksPass(MPM);
@@ -1953,6 +1961,9 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
invokeFullLinkTimeOptimizationLastEPCallbacks(MPM, Level);
+ if (EnableMergeFuncIgnoringConst)
+ MPM.addPass(MergeFuncIgnoringConstPass());
+
// Emit annotation remarks.
addAnnotationRemarksPass(MPM);
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index df9f14920f29161..fe6837b3891aead 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -84,6 +84,7 @@ MODULE_PASS("lower-ifunc", LowerIFuncPass())
MODULE_PASS("lowertypetests", LowerTypeTestsPass())
MODULE_PASS("metarenamer", MetaRenamerPass())
MODULE_PASS("mergefunc", MergeFunctionsPass())
+MODULE_PASS("mergefunc-ignoring-const", MergeFuncIgnoringConstPass())
MODULE_PASS("name-anon-globals", NameAnonGlobalPass())
MODULE_PASS("no-op-module", NoOpModulePass())
MODULE_PASS("objc-arc-apelim", ObjCARCAPElimPass())
diff --git a/llvm/lib/Transforms/IPO/CMakeLists.txt b/llvm/lib/Transforms/IPO/CMakeLists.txt
index 034f1587ae8df44..4dac04d3369950f 100644
--- a/llvm/lib/Transforms/IPO/CMakeLists.txt
+++ b/llvm/lib/Transforms/IPO/CMakeLists.txt
@@ -30,6 +30,7 @@ add_llvm_component_library(LLVMipo
LowerTypeTests.cpp
MemProfContextDisambiguation.cpp
MergeFunctions.cpp
+ MergeFunctionsIgnoringConst.cpp
ModuleInliner.cpp
OpenMPOpt.cpp
PartialInlining.cpp
diff --git a/llvm/lib/Transforms/IPO/MergeFunctionsIgnoringConst.cpp b/llvm/lib/Transforms/IPO/MergeFunctionsIgnoringConst.cpp
new file mode 100644
index 000000000000000..4c576123a11f365
--- /dev/null
+++ b/llvm/lib/Transforms/IPO/MergeFunctionsIgnoringConst.cpp
@@ -0,0 +1,1430 @@
+//===--- MergeFunctionsIgnoringConst.cpp - Merge functions ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass looks for similar functions that are mergeable and folds them.
+// The implementation is similar to LLVM's MergeFunctions pass. Instead of
+// merging identical functions, it merges functions which only differ by a few
+// constants in certain instructions.
+// This is copied from Swift's implementation.
+// TODO: We should generalize this pass and share it with Swift's
+// implementation.
+//
+// This pass should run after LLVM's MergeFunctions pass, because it works best
+// if there are no _identical_ functions in the module.
+// Note: it would also work for identical functions but could produce more
+// code overhead than the LLVM pass.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/IPO/MergeFunctionsIgnoringConst.h"
+// #include "llvm/Transforms/Utils/GlobalMergeFunctions.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Transforms/Utils/FunctionComparatorIgnoringConst.h"
+// #include "llvm/ADT/Triple.h"
+#include "llvm/Analysis/ObjCARCUtil.h"
+#include "llvm/CodeGen/StableHashing.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+// #include "llvm/IR/GlobalPtrAuthInfo.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InlineAsm.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/StructuralHash.h"
+#include "llvm/IR/ValueHandle.h"
+#include "llvm/IR/ValueMap.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/IPO.h"
+#include <vector>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "mergefunc-ignoring-const"
+
+STATISTIC(NumFunctionsMergedIgnoringConst, "Number of functions merged");
+STATISTIC(NumThunksWrittenIgnoringConst, "Number of thunks generated");
+
+static cl::opt<bool>
+ EnableMergeFunc2("enable-merge-func2", cl::init(false), cl::Hidden,
+ cl::desc("Enable more aggressive function merger"));
+
+static cl::opt<unsigned> NumFunctionsIgnoringConstForSanityCheck(
+ "mergefunc-ignoringconst-sanity",
+ cl::desc("How many functions in module could be used for "
+ "MergeFunctionsIgnoringConst pass sanity check. "
+ "'0' disables this check. Works only with '-debug' key."),
+ cl::init(0), cl::Hidden);
+
+static cl::opt<unsigned> IgnoringConstMergeThreshold(
+ "mergefunc-ignoringconst-threshold",
+ cl::desc("Functions larger than the threshold are considered for merging."
+ "'0' disables function merging at all."),
+ cl::init(15), cl::Hidden);
+
+cl::opt<bool> UseLinkOnceODRLinkageMerging(
+ "use-linkonceodr-linkage-merging", cl::init(false), cl::Hidden,
+ cl::desc(
+ "Use LinkeOnceODR linkage to deduplicate the identical merged function "
+ "(default = off)"));
+
+cl::opt<bool> NoInlineForMergedFunction(
+ "no-inline-merged-function", cl::init(false), cl::Hidden,
+ cl::desc("set noinline for merged function (default = off)"));
+
+static cl::opt<bool>
+ CastArrayType("merge-cast-array-type", cl::init(false), cl::Hidden,
+ cl::desc("support for casting array type (default = off)"));
+
+static cl::opt<bool> IgnoreMusttailFunction(
+ "ignore-musttail-function", cl::init(false), cl::Hidden,
+ cl::desc(
+ "ignore functions containing callsites with musttail (default = off)"));
+
+static cl::opt<bool> AlwaysCallThunk(
+ "merge-always-call-thunk", cl::init(false), cl::Hidden,
+ cl::desc(
+ "do not replace callsites and always emit a thunk (default = off)"));
+
+static cl::list<std::string> MergeBlockRegexFilters(
+ "merge-block-regex", cl::Optional,
+ cl::desc("Block functions from merging if they match the given "
+ "regular expression"),
+ cl::ZeroOrMore);
+
+static cl::list<std::string> MergeAllowRegexFilters(
+ "merge-allow-regex", cl::Optional,
+ cl::desc("Allow functions from merging if they match the given "
+ "regular expression"),
+ cl::ZeroOrMore);
+
+bool isEligibleInstrunctionForConstantSharing(const Instruction *I) {
+ switch (I->getOpcode()) {
+ case Instruction::Load:
+ case Instruction::Store:
+ case Instruction::Call:
+ return true;
+ default: {
+ if (EnableMergeFunc2 && I->getOpcode() == Instruction::Invoke)
+ return true;
+ return false;
+ }
+ }
+}
+
+/// Returns true if the \opIdx operand of \p CI is the callee operand.
+static bool isCalleeOperand(const CallBase *CI, unsigned opIdx) {
+ return &CI->getCalledOperandUse() == &CI->getOperandUse(opIdx);
+}
+
+static bool canParameterizeCallOperand(const CallBase *CI, unsigned opIdx) {
+ if (CI->isInlineAsm())
+ return false;
+ Function *Callee = CI->getCalledOperand()
+ ? dyn_cast_or_null<Function>(
+ CI->getCalledOperand()->stripPointerCasts())
+ : nullptr;
+ if (Callee) {
+ if (Callee->isIntrinsic())
+ return false;
+ // objc_msgSend stubs must be called, and can't have their address taken.
+ if (Callee->getName().startswith("objc_msgSend$"))
+ return false;
+ }
+ if (isCalleeOperand(CI, opIdx) &&
+ CI->getOperandBundle(LLVMContext::OB_ptrauth).has_value()) {
+ // The operand is the callee and it has already been signed. Ignore this
+ // because we cannot add another ptrauth bundle to the call instruction.
+ return false;
+ }
+ return true;
+}
+
+bool isEligibleOperandForConstantSharing(const Instruction *I, unsigned OpIdx) {
+ assert(OpIdx < I->getNumOperands() && "Invalid operand index");
+
+ if (!isEligibleInstrunctionForConstantSharing(I))
+ return false;
+
+ auto Opnd = I->getOperand(OpIdx);
+ if (!isa<Constant>(Opnd))
+ return false;
+
+ if (const auto *CI = dyn_cast<CallBase>(I))
+ return canParameterizeCallOperand(CI, OpIdx);
+
+ return true;
+}
+
+namespace {
+
+/// MergeFuncIgnoringConst finds functions which only differ by constants in
+/// certain instructions, e.g. resulting from specialized functions of layout
+/// compatible types.
+/// Such functions are merged by replacing the differing constants by a
+/// parameter. The original functions are replaced by thunks which call the
+/// merged function with the specific argument constants.
+///
+class MergeFuncIgnoringConstImpl { // : public ModulePass {
+public:
+ MergeFuncIgnoringConstImpl() : FnTree(FunctionNodeCmp(&GlobalNumbers)) {}
+
+ MergeFuncIgnoringConstImpl(bool ptrAuthEnabled, unsigned ptrAuthKey)
+ : FnTree(FunctionNodeCmp(&GlobalNumbers)), ptrAuthOptionsSet(true),
+ ptrAuthEnabled(ptrAuthEnabled), ptrAuthKey(ptrAuthKey) {}
+
+ bool runImpl(Module &M);
+
+private:
+ struct FunctionEntry;
+
+ /// Describes the set of functions which are considered as "equivalent" (i.e.
+ /// only differing by some constants).
+ struct EquivalenceClass {
+ /// The single-linked list of all functions which are a member of this
+ /// equivalence class.
+ FunctionEntry *First;
+
+ /// A very cheap hash, used to early exit if functions do not match.
+ llvm::IRHash Hash;
+
+ public:
+ // Note the hash is recalculated potentially multiple times, but it is
+ // cheap.
+ EquivalenceClass(FunctionEntry *First)
+ : First(First), Hash(StructuralHash(*First->F)) {
+ assert(!First->Next);
+ }
+ };
+
+ /// The function comparison operator is provided here so that FunctionNodes do
+ /// not need to become larger with another pointer.
+ class FunctionNodeCmp {
+ GlobalNumberState *GlobalNumbers;
+
+ public:
+ FunctionNodeCmp(GlobalNumberState *GN) : GlobalNumbers(GN) {}
+ bool operator()(const EquivalenceClass &LHS,
+ const EquivalenceClass &RHS) const {
+ // Order first by hashes, then full function comparison.
+ if (LHS.Hash != RHS.Hash)
+ return LHS.Hash < RHS.Hash;
+ FunctionComparatorIgnoringConst FCmp(LHS.First->F, RHS.First->F,
+ GlobalNumbers);
+ return FCmp.compareIgnoringConsts() == -1;
+ }
+ };
+ using FnTreeType = std::set<EquivalenceClass, FunctionNodeCmp>;
+
+ ///
+ struct FunctionEntry {
+ FunctionEntry(Function *F, FnTreeType::iterator I)
+ : F(F), Next(nullptr), numUnhandledCallees(0), TreeIter(I),
+ isMerged(false) {}
+
+ /// Back-link to the function.
+ AssertingVH<Function> F;
+
+ /// The next function in its equivalence class.
+ FunctionEntry *Next;
+
+ /// The number of not-yet merged callees. Used to process the merging in
+ /// bottom-up call order.
+ /// This is only valid in the first entry of an equivalence class. The
+ /// counts of all functions in an equivalence class are accumulated in the
+ /// first entry.
+ int numUnhandledCallees;
+
+ /// The iterator of the function's equivalence class in the FnTree.
+ /// It's FnTree.end() if the function is not in an equivalence class.
+ FnTreeType::iterator TreeIter;
+
+ /// True if this function is already a thunk, calling the merged function.
+ bool isMerged;
+ };
+
+ /// Describes an operator of a specific instruction.
+ struct OpLocation {
+ Instruction *I;
+ unsigned OpIndex;
+ };
+
+ /// Information for a function. Used during merging.
+ struct FunctionInfo {
+
+ FunctionInfo(Function *F)
+ : F(F), CurrentInst(nullptr), NumParamsNeeded(0) {}
+
+ void init() {
+ CurrentInst = &*F->begin()->begin();
+ NumParamsNeeded = 0;
+ }
+
+ /// Advances the current instruction to the next instruction.
+ void nextInst() {
+ assert(CurrentInst);
+ if (CurrentInst->isTerminator()) {
+ auto BlockIter = std::next(CurrentInst->getParent()->getIterator());
+ if (BlockIter == F->end()) {
+ CurrentInst = nullptr;
+ return;
+ }
+ CurrentInst = &*BlockIter->begin();
+ return;
+ }
+ CurrentInst = &*std::next(CurrentInst->getIterator());
+ }
+
+ /// Returns true if the operand \p OpIdx of the current instruction is the
+ /// callee of a call, which needs to be signed if passed as a parameter.
+ bool needsPointerSigning(unsigned OpIdx) const {
+ if (auto *CI = dyn_cast<CallInst>(CurrentInst))
+ return isCalleeOperand(CI, OpIdx);
+ return false;
+ }
+
+ Function *F;
+
+ /// The current instruction while iterating over all instructions.
+ Instruction *CurrentInst;
+
+ /// Roughly the number of parameters needed if this function would be
+ /// merged with the first function of the equivalence class.
+ int NumParamsNeeded;
+ };
+
+ using FunctionInfos = SmallVector<FunctionInfo, 8>;
+
+ /// Describes a parameter which we create to parameterize the merged function.
+ struct ParamInfo {
+ /// The value of the parameter for all the functions in the equivalence
+ /// class.
+ SmallVector<Constant *, 8> Values;
+
+ /// All uses of the parameter in the merged function.
+ SmallVector<OpLocation, 16> Uses;
+
+ /// The discriminator for pointer signing.
+ /// Only not null if needsPointerSigning is true.
+ ConstantInt *discriminator = nullptr;
+
+ /// True if the value is a callee function, which needs to be signed if
+ /// passed as a parameter.
+ bool needsPointerSigning = false;
+
+ /// Checks if this parameter can be used to describe an operand in all
+ /// functions of the equivalence class. Returns true if all values match
+ /// the specific instruction operands in all functions.
+ bool matches(const FunctionInfos &FInfos, unsigned OpIdx,
+ bool ptrAuthEnabled) const {
+ unsigned NumFuncs = FInfos.size();
+ assert(Values.size() == NumFuncs);
+ if (ptrAuthEnabled &&
+ needsPointerSigning != FInfos[0].needsPointerSigning(OpIdx)) {
+ return false;
+ }
+ for (unsigned Idx = 0; Idx < NumFuncs; ++Idx) {
+ const FunctionInfo &FI = FInfos[Idx];
+ Constant *C = cast<Constant>(FI.CurrentInst->getOperand(OpIdx));
+ if (Values[Idx] != C)
+ return false;
+ }
+ return true;
+ }
+
+ /// Computes the discriminator for pointer signing.
+ void computeDiscriminator(LLVMContext &Context) {
+ assert(needsPointerSigning);
+ assert(!discriminator);
+
+ /// Get a hash from the concatenated function names.
+ /// The hash is deterministic, because the order of values depends on the
+ /// order of functions in the module, which is itself deterministic.
+ /// Note that the hash is not part of the ABI, because it's purly used
+ /// for pointer authentication between a module-private caller-callee
+ /// pair.
+ std::string concatenatedCalleeNames;
+ for (Constant *value : Values) {
+ if (auto *GO = dyn_cast<GlobalObject>(value))
+ concatenatedCalleeNames += GO->getName();
+ }
+ uint64_t rawHash = stable_hash_combine_string(concatenatedCalleeNames);
+ IntegerType *discrTy = Type::getInt64Ty(Context);
+ discriminator = ConstantInt::get(discrTy, (rawHash % 0xFFFF) + 1);
+ }
+ };
+
+ using ParamInfos = SmallVector<ParamInfo, 16>;
+
+ Module *module = nullptr;
+ ModuleSummaryIndex *ExportSummary;
+ const ModuleSummaryIndex *ImportSummary;
+
+ GlobalNumberState GlobalNumbers;
+
+ /// A work queue of functions that may have been modified and should be
+ /// analyzed again.
+ std::vector<WeakTrackingVH> Deferred;
+
+ /// The set of all distinct functions. Use the insert() and remove() methods
+ /// to modify it. The map allows efficient lookup and deferring of Functions.
+ FnTreeType FnTree;
+
+ ValueMap<Function *, FunctionEntry *> FuncEntries;
+
+ // Maps a function-pointer / discriminator pair to a corresponding global in
+ // the llvm.ptrauth section.
+ // This map is used as a cache to not create ptrauth globals twice.
+ DenseMap<std::pair<Constant *, ConstantInt *>, Constant *> ptrAuthGlobals;
+
+ /// If true, ptrAuthEnabled and ptrAuthKey are valid.
+ bool ptrAuthOptionsSet = false;
+
+ /// True if the architecture has pointer authentication enabled.
+ bool ptrAuthEnabled = false;
+
+ /// The key for pointer authentication.
+ unsigned ptrAuthKey = 0;
+
+ FunctionEntry *getEntry(Function *F) const { return FuncEntries.lookup(F); }
+
+ bool isInEquivalenceClass(FunctionEntry *FE) const {
+ if (FE->TreeIter != FnTree.end()) {
+ return true;
+ }
+ assert(!FE->Next);
+ assert(FE->numUnhandledCallees == 0);
+ return false;
+ }
+
+ /// Checks the rules of order relation introduced among functions set.
+ /// Returns true, if sanity check has been passed, and false if failed.
+ bool doSanityCheck(std::vector<WeakTrackingVH> &Worklist);
+
+ /// Updates the numUnhandledCallees of all user functions of the equivalence
+ /// class containing \p FE by \p Delta.
+ void updateUnhandledCalleeCount(FunctionEntry *FE, int Delta);
+
+ bool tryMergeEquivalenceClass(FunctionEntry *FirstInClass);
+
+ FunctionInfo removeFuncWithMostParams(FunctionInfos &FInfos);
+
+ bool deriveParams(ParamInfos &Params, FunctionInfos &FInfos,
+ unsigned maxParams);
+
+ bool numOperandsDiffer(FunctionInfos &FInfos);
+
+ bool constsDiffer(const FunctionInfos &FInfos, unsigned OpIdx);
+
+ bool tryMapToParameter(FunctionInfos &FInfos, unsigned OpIdx,
+ ParamInfos &Params, unsigned maxParams);
+
+ void replaceCallWithAddedPtrAuth(CallInst *origCall, Value *newCallee,
+ ConstantInt *discriminator);
+
+ void mergeWithParams(const FunctionInfos &FInfos, ParamInfos &Params);
+ static void dumpMergeInfo(const FunctionInfos &FInfos, unsigned);
+
+ void removeEquivalenceClassFromTree(FunctionEntry *FE);
+
+ void writeThunk(Function *ToFunc, Function *Thunk, const ParamInfos &Params,
+ unsigned FuncIdx);
+
+ bool isPtrAuthEnabled() const {
+ // TODO: fix pointer authentication
+ // assert(ptrAuthOptionsSet);
+ return ptrAuthEnabled;
+ }
+
+ ConstantInt *getPtrAuthKey() {
+ // TODO: fix pointer authentication
+ // assert(isPtrAuthEnabled());
+ return ConstantInt::get(Type::getInt32Ty(module->getContext()), ptrAuthKey);
+ }
+
+ /// Returns the value of function \p FuncIdx, and signes it if required.
+ Constant *getSignedValue(const ParamInfo &PI, unsigned FuncIdx) {
+ Constant *value = PI.Values[FuncIdx];
+ if (!PI.needsPointerSigning)
+ return value;
+
+ auto lookupKey = std::make_pair(value, PI.discriminator);
+ Constant *&ptrAuthGlobal = ptrAuthGlobals[lookupKey];
+ if (!ptrAuthGlobal) {
+#if 0
+ ptrAuthGlobal = GlobalPtrAuthInfo::create(
+ *module, value, getPtrAuthKey(),
+ ConstantInt::get(PI.discriminator->getType(), 0), PI.discriminator);
+#endif
+ }
+ return ptrAuthGlobal;
+ }
+
+ /// Replace all direct calls of Old with calls of New. Will bitcast New if
+ /// necessary to make types match.
+ bool replaceDirectCallers(Function *Old, Function *New,
+ const ParamInfos &Params, unsigned FuncIdx);
+};
+
+#if 0
+class MergeFuncIgnoringConst : public ModulePass {
+public:
+ static char ID;
+ /// True if the architecture has pointer authentication enabled.
+ bool ptrAuthEnabled = false;
+
+ /// The key for pointer authentication.
+ unsigned ptrAuthKey = 0;
+ ModuleSummaryIndex *ExportSummary;
+ const ModuleSummaryIndex *ImportSummary;
+
+ MergeFuncIgnoringConst() : ModulePass(ID) {
+ initializeMergeFuncIgnoringConstPass(*llvm::PassRegistry::getPassRegistry());
+ }
+ MergeFuncIgnoringConst(bool ptrAuthEnabled, unsigned ptrAuthKey)
+ : ModulePass(ID), ptrAuthEnabled(ptrAuthEnabled), ptrAuthKey(ptrAuthKey) {
+ initializeMergeFuncIgnoringConstPass(*llvm::PassRegistry::getPassRegistry());
+ }
+ bool runOnModule(Module &M) override;
+};
+#endif
+
+} // end anonymous namespace
+
+#if 0
+char MergeFuncIgnoringConst::ID = 0;
+INITIALIZE_PASS_BEGIN(MergeFuncIgnoringConst, "merge-func-ignoring-const",
+ "merge function pass ignoring const", false, false)
+INITIALIZE_PASS_END(MergeFuncIgnoringConst, "merge-func-ignoring-const",
+ "merge function pass ignoring const", false, false)
+#endif
+bool MergeFuncIgnoringConstImpl::doSanityCheck(
+ std::vector<WeakTrackingVH> &Worklist) {
+ if (const unsigned Max = NumFunctionsIgnoringConstForSanityCheck) {
+ unsigned TripleNumber = 0;
+ bool Valid = true;
+
+ dbgs() << "MERGEFUNC-SANITY: Started for first " << Max << " functions.\n";
+
+ unsigned i = 0;
+ for (std::vector<WeakTrackingVH>::iterator I = Worklist.begin(),
+ E = Worklist.end();
+ I != E && i < Max; ++I, ++i) {
+ unsigned j = i;
+ for (std::vector<WeakTrackingVH>::iterator J = I; J != E && j < Max;
+ ++J, ++j) {
+ Function *F1 = cast<Function>(*I);
+ Function *F2 = cast<Function>(*J);
+ int Res1 = FunctionComparatorIgnoringConst(F1, F2, &GlobalNumbers)
+ .compareIgnoringConsts();
+ int Res2 = FunctionComparatorIgnoringConst(F2, F1, &GlobalNumbers)
+ .compareIgnoringConsts();
+
+ // If F1 <= F2, then F2 >= F1, otherwise report failure.
+ if (Res1 != -Res2) {
+ dbgs() << "MERGEFUNC-SANITY: Non-symmetric; triple: " << TripleNumber
+ << "\n";
+ LLVM_DEBUG(F1->dump());
+ LLVM_DEBUG(F2->dump());
+ Valid = false;
+ }
+
+ if (Res1 == 0)
+ continue;
+
+ unsigned k = j;
+ for (std::vector<WeakTrackingVH>::iterator K = J; K != E && k < Max;
+ ++k, ++K, ++TripleNumber) {
+ if (K == J)
+ continue;
+
+ Function *F3 = cast<Function>(*K);
+ int Res3 = FunctionComparatorIgnoringConst(F1, F3, &GlobalNumbers)
+ .compareIgnoringConsts();
+ int Res4 = FunctionComparatorIgnoringConst(F2, F3, &GlobalNumbers)
+ .compareIgnoringConsts();
+
+ bool Transitive = true;
+
+ if (Res1 != 0 && Res1 == Res4) {
+ // F1 > F2, F2 > F3 => F1 > F3
+ Transitive = Res3 == Res1;
+ } else if (Res3 != 0 && Res3 == -Res4) {
+ // F1 > F3, F3 > F2 => F1 > F2
+ Transitive = Res3 == Res1;
+ } else if (Res4 != 0 && -Res3 == Res4) {
+ // F2 > F3, F3 > F1 => F2 > F1
+ Transitive = Res4 == -Res1;
+ }
+
+ if (!Transitive) {
+ dbgs() << "MERGEFUNC-SANITY: Non-transitive; triple: "
+ << TripleNumber << "\n";
+ dbgs() << "Res1, Res3, Res4: " << Res1 << ", " << Res3 << ", "
+ << Res4 << "\n";
+ LLVM_DEBUG(F1->dump());
+ LLVM_DEBUG(F2->dump());
+ LLVM_DEBUG(F3->dump());
+ Valid = false;
+ }
+ }
+ }
+ }
+
+ dbgs() << "MERGEFUNC-SANITY: " << (Valid ? "Passed." : "Failed.") << "\n";
+ return Valid;
+ }
+ return true;
+}
+
+/// Returns true if functions containing calls to \p F may be merged together.
+static bool mayMergeCallsToFunction(Function &F) {
+ StringRef Name = F.getName();
+
+ // Calls to dtrace probes must generate unique patchpoints.
+ if (Name.startswith("__dtrace"))
+ return false;
+
+ return true;
+}
+
+/// Returns the benefit, which is approximately the size of the function.
+/// Return 0, if the function should not be merged.
+static unsigned getBenefit(Function *F) {
+ unsigned Benefit = 0;
+
+ // We don't want to merge very small functions, because the overhead of
+ // adding creating thunks and/or adding parameters to the call sites
+ // outweighs the benefit.
+ for (BasicBlock &BB : *F) {
+ for (Instruction &I : BB) {
+ if (CallBase *CB = dyn_cast<CallBase>(&I)) {
+ Function *Callee = CB->getCalledFunction();
+ if (Callee && !mayMergeCallsToFunction(*Callee))
+ return 0;
+ if (!Callee || !Callee->isIntrinsic()) {
+ Benefit += 5;
+ continue;
+ }
+ }
+ Benefit += 1;
+ }
+ }
+ return Benefit;
+}
+
+/// Returns true if function \p F is eligible for merging.
+bool isEligibleFunction(Function *F) {
+ if (F->isDeclaration())
+ return false;
+
+ if (F->hasFnAttribute(llvm::Attribute::NoMerge))
+ return false;
+
+ if (F->hasAvailableExternallyLinkage()) {
+ return false;
+ }
+
+ if (F->getFunctionType()->isVarArg()) {
+ return false;
+ }
+
+ // Check against blocklist.
+ if (!MergeBlockRegexFilters.empty()) {
+ StringRef FuncName = F->getName();
+ for (const auto &tRegex : MergeBlockRegexFilters)
+ if (Regex(tRegex).match(FuncName)) {
+ return false;
+ }
+ }
+ // Check against allowlist
+ if (!MergeAllowRegexFilters.empty()) {
+ StringRef FuncName = F->getName();
+ bool found = false;
+ for (const auto &tRegex : MergeAllowRegexFilters)
+ if (Regex(tRegex).match(FuncName)) {
+ found = true;
+ break;
+ }
+ if (!found)
+ return false;
+ }
+
+ if (F->getCallingConv() == CallingConv::SwiftTail)
+ return false;
+
+ // if function contains callsites with musttail, if we merge
+ // it, the merged function will have the musttail callsite, but
+ // the number of parameters can change, thus the parameter count
+ // of the callsite will mismatch with the function itself.
+ if (IgnoreMusttailFunction) {
+ for (const BasicBlock &BB : *F) {
+ for (const Instruction &I : BB) {
+ const auto *CB = dyn_cast<CallBase>(&I);
+ if (CB && CB->isMustTailCall())
+ return false;
+ }
+ }
+ }
+
+ unsigned Benefit = getBenefit(F);
+ if (Benefit < IgnoringConstMergeThreshold) {
+ return false;
+ }
+
+ return true;
+}
+
+static bool runInternal(Module &M) {
+ return MergeFuncIgnoringConstImpl().runImpl(M);
+}
+
+// bool MergeFuncIgnoringConst::runOnModule(Module &M) { return runInternal(M);
+// }
+
+bool MergeFuncIgnoringConstImpl::runImpl(Module &M) {
+ if (IgnoringConstMergeThreshold == 0)
+ return false;
+
+ module = &M;
+
+#if 0
+ // TODO: fix pointer authentication
+ if (!ptrAuthOptionsSet) {
+ // If invoked from IRGen in the compiler, those options are already set.
+ // If invoked from swift-llvm-opt, derive the options from the target triple.
+ Triple triple(M.getTargetTriple());
+ ptrAuthEnabled = (triple.getSubArch() == Triple::AArch64SubArch_arm64e);
+ ptrAuthKey = (unsigned)clang::PointerAuthSchema::ARM8_3Key::ASIA;
+ ptrAuthOptionsSet = true;
+ }
+#endif
+
+ bool Changed = false;
+
+ // All functions in the module, ordered by hash. Functions with a unique
+ // hash value are easily eliminated.
+ std::vector<std::pair<llvm::IRHash, Function *>> HashedFuncs;
+
+ for (Function &Func : M) {
+ if (isEligibleFunction(&Func)) {
+ HashedFuncs.push_back({StructuralHash(Func), &Func});
+ }
+ }
+
+ std::stable_sort(HashedFuncs.begin(), HashedFuncs.end(),
+ [](const std::pair<llvm::IRHash, Function *> &a,
+ const std::pair<llvm::IRHash, Function *> &b) {
+ return a.first < b.first;
+ });
+
+ std::vector<FunctionEntry> FuncEntryStorage;
+ FuncEntryStorage.reserve(HashedFuncs.size());
+
+ auto S = HashedFuncs.begin();
+ for (auto I = HashedFuncs.begin(), IE = HashedFuncs.end(); I != IE; ++I) {
+
+ Function *F = I->second;
+ FuncEntryStorage.push_back(FunctionEntry(F, FnTree.end()));
+ FunctionEntry &FE = FuncEntryStorage.back();
+ FuncEntries[F] = &FE;
+
+ // If the hash value matches the previous value or the next one, we must
+ // consider merging it. Otherwise it is dropped and never considered again.
+ if ((I != S && std::prev(I)->first == I->first) ||
+ (std::next(I) != IE && std::next(I)->first == I->first)) {
+ Deferred.push_back(WeakTrackingVH(F));
+ }
+ }
+
+ do {
+ std::vector<WeakTrackingVH> Worklist;
+ Deferred.swap(Worklist);
+
+ LLVM_DEBUG(dbgs() << "======\nbuild tree: worklist-size=" << Worklist.size()
+ << '\n');
+ LLVM_DEBUG(doSanityCheck(Worklist));
+
+ SmallVector<FunctionEntry *, 8> FuncsToMerge;
+
+ // Insert all candidates into the Worklist.
+ for (WeakTrackingVH &I : Worklist) {
+ if (!I)
+ continue;
+ Function *F = cast<Function>(I);
+ FunctionEntry *FE = getEntry(F);
+ assert(!isInEquivalenceClass(FE));
+
+ std::pair<FnTreeType::iterator, bool> Result = FnTree.insert(FE);
+
+ FE->TreeIter = Result.first;
+ const EquivalenceClass &Eq = *Result.first;
+
+ if (Result.second) {
+ assert(Eq.First == FE);
+ LLVM_DEBUG(dbgs() << " new in tree: " << F->getName() << '\n');
+ } else {
+ assert(Eq.First != FE);
+ LLVM_DEBUG(dbgs() << " add to existing: " << F->getName() << '\n');
+ // Add the function to the existing equivalence class.
+ FE->Next = Eq.First->Next;
+ Eq.First->Next = FE;
+ // Schedule for merging if the function's equivalence class reaches the
+ // size of 2.
+ if (!FE->Next)
+ FuncsToMerge.push_back(Eq.First);
+ }
+ }
+ LLVM_DEBUG(dbgs() << "merge functions: tree-size=" << FnTree.size()
+ << '\n');
+
+ // Figure out the leaf functions. We want to do the merging in bottom-up
+ // call order. This ensures that we don't parameterize on callee function
+ // names if we don't have to (because the callee may be merged).
+ // Note that "leaf functions" refer to the sub-call-graph of functions which
+ // are in the FnTree.
+ for (FunctionEntry *ToMerge : FuncsToMerge) {
+ assert(isInEquivalenceClass(ToMerge));
+ updateUnhandledCalleeCount(ToMerge, 1);
+ }
+
+ // Check if there are any leaf functions at all.
+ bool LeafFound = false;
+ for (FunctionEntry *ToMerge : FuncsToMerge) {
+ if (ToMerge->numUnhandledCallees == 0)
+ LeafFound = true;
+ }
+ for (FunctionEntry *ToMerge : FuncsToMerge) {
+ if (isInEquivalenceClass(ToMerge)) {
+ // Only merge leaf functions (or all functions if all functions are in
+ // a call cycle).
+ if (ToMerge->numUnhandledCallees == 0 || !LeafFound) {
+ updateUnhandledCalleeCount(ToMerge, -1);
+ Changed |= tryMergeEquivalenceClass(ToMerge);
+ } else {
+ // Non-leaf functions (i.e. functions in a call cycle) may become
+ // leaf functions in the next iteration.
+ removeEquivalenceClassFromTree(ToMerge);
+ }
+ }
+ }
+ } while (!Deferred.empty());
+
+ FnTree.clear();
+ GlobalNumbers.clear();
+ FuncEntries.clear();
+ ptrAuthGlobals.clear();
+
+ return Changed;
+}
+
+void MergeFuncIgnoringConstImpl::updateUnhandledCalleeCount(FunctionEntry *FE,
+ int Delta) {
+ // Iterate over all functions of FE's equivalence class.
+ do {
+ for (Use &U : FE->F->uses()) {
+ if (auto *I = dyn_cast<Instruction>(U.getUser())) {
+ FunctionEntry *CallerFE = getEntry(I->getFunction());
+ if (CallerFE && CallerFE->TreeIter != FnTree.end()) {
+ // Accumulate the count in the first entry of the equivalence class.
+ FunctionEntry *Head = CallerFE->TreeIter->First;
+ Head->numUnhandledCallees += Delta;
+ }
+ }
+ }
+ FE = FE->Next;
+ } while (FE);
+}
+
+bool MergeFuncIgnoringConstImpl::tryMergeEquivalenceClass(
+ FunctionEntry *FirstInClass) {
+ // Build the FInfos vector from all functions in the equivalence class.
+ FunctionInfos FInfos;
+ FunctionEntry *FE = FirstInClass;
+ do {
+ FInfos.push_back(FunctionInfo(FE->F));
+ FE->isMerged = true;
+ FE = FE->Next;
+ } while (FE);
+ assert(FInfos.size() >= 2);
+
+ // Merged or not: in any case we remove the equivalence class from the FnTree.
+ removeEquivalenceClassFromTree(FirstInClass);
+
+ // Contains functions which differ too much from the first function (i.e.
+ // would need too many parameters).
+ FunctionInfos Removed;
+
+ bool Changed = false;
+ int Try = 0;
+
+ unsigned Benefit = getBenefit(FirstInClass->F);
+
+ // The bigger the function, the more parameters are allowed.
+ unsigned maxParams = std::max(4u, Benefit / 100);
+
+ // We need multiple tries if there are some functions in FInfos which differ
+ // too much from the first function in FInfos. But we limit the number of
+ // tries to a small number, because this is quadratic.
+ while (FInfos.size() >= 2 && Try++ < 4) {
+ ParamInfos Params;
+ bool Merged = deriveParams(Params, FInfos, maxParams);
+ if (Merged) {
+ mergeWithParams(FInfos, Params);
+ Changed = true;
+ } else {
+ // We ran out of parameters. Remove the function from the set which
+ // differs most from the first function.
+ Removed.push_back(removeFuncWithMostParams(FInfos));
+ }
+ if (Merged || FInfos.size() < 2) {
+ // Try again with the functions which were removed from the original set.
+ FInfos.swap(Removed);
+ Removed.clear();
+ }
+ }
+ return Changed;
+}
+
+/// Remove the function from \p FInfos which needs the most parameters. Add the
+/// removed function to
+MergeFuncIgnoringConstImpl::FunctionInfo
+MergeFuncIgnoringConstImpl::removeFuncWithMostParams(FunctionInfos &FInfos) {
+ FunctionInfos::iterator MaxIter = FInfos.end();
+ for (auto Iter = FInfos.begin(), End = FInfos.end(); Iter != End; ++Iter) {
+ if (MaxIter == FInfos.end() ||
+ Iter->NumParamsNeeded > MaxIter->NumParamsNeeded) {
+ MaxIter = Iter;
+ }
+ }
+ FunctionInfo Removed = *MaxIter;
+ FInfos.erase(MaxIter);
+ return Removed;
+}
+
+/// Finds the set of parameters which are required to merge the functions in
+/// \p FInfos.
+/// Returns true on success, i.e. the functions in \p FInfos can be merged with
+/// the parameters returned in \p Params.
+bool MergeFuncIgnoringConstImpl::deriveParams(ParamInfos &Params,
+ FunctionInfos &FInfos,
+ unsigned maxParams) {
+ for (FunctionInfo &FI : FInfos)
+ FI.init();
+
+ FunctionInfo &FirstFI = FInfos.front();
+
+ // Iterate over all instructions synchronously in all functions.
+ do {
+ if (isEligibleInstrunctionForConstantSharing(FirstFI.CurrentInst)) {
+
+ // Here we handle a rare corner case which needs to be explained:
+ // Usually the number of operands match, because otherwise the functions
+ // in FInfos would not be in the same equivalence class. There is only one
+ // exception to that: If the current instruction is a call to a function,
+ // which was merged in the previous iteration (in
+ // tryMergeEquivalenceClass) then the call could be replaced and has more
+ // arguments than the original call.
+ if (numOperandsDiffer(FInfos)) {
+ assert(isa<CallInst>(FirstFI.CurrentInst) &&
+ "only calls are expected to differ in number of operands");
+ return false;
+ }
+
+ for (unsigned OpIdx = 0, NumOps = FirstFI.CurrentInst->getNumOperands();
+ OpIdx != NumOps; ++OpIdx) {
+
+ if (constsDiffer(FInfos, OpIdx)) {
+ // This instruction has operands which differ in at least some
+ // functions. So we need to parameterize it.
+ if (!tryMapToParameter(FInfos, OpIdx, Params, maxParams)) {
+ // We ran out of parameters.
+ return false;
+ }
+ }
+ }
+ }
+ // Go to the next instruction in all functions.
+ for (FunctionInfo &FI : FInfos)
+ FI.nextInst();
+ } while (FirstFI.CurrentInst);
+
+ return true;
+}
+
+/// Returns true if the number of operands of the current instruction differs.
+bool MergeFuncIgnoringConstImpl::numOperandsDiffer(FunctionInfos &FInfos) {
+ unsigned numOps = FInfos[0].CurrentInst->getNumOperands();
+ for (const FunctionInfo &FI : ArrayRef<FunctionInfo>(FInfos).drop_front(1)) {
+ if (FI.CurrentInst->getNumOperands() != numOps)
+ return true;
+ }
+ return false;
+}
+
+/// Returns true if the \p OpIdx's constant operand in the current instruction
+/// does differ in any of the functions in \p FInfos.
+bool MergeFuncIgnoringConstImpl::constsDiffer(const FunctionInfos &FInfos,
+ unsigned OpIdx) {
+ Constant *CommonConst = nullptr;
+
+ for (const FunctionInfo &FI : FInfos) {
+ Value *Op = FI.CurrentInst->getOperand(OpIdx);
+ if (auto *C = dyn_cast<Constant>(Op)) {
+ if (!CommonConst) {
+ CommonConst = C;
+ } else if (EnableMergeFunc2 && isa<ConstantPointerNull>(CommonConst) &&
+ isa<ConstantPointerNull>(C)) {
+ // if both are null pointer, and if they are different constants
+ // due to type, still treat them as the same.
+ } else if (C != CommonConst) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+/// Create a new parameter for differing operands or try to reuse an existing
+/// parameter.
+/// Returns true if a parameter could be created or found without exceeding the
+/// maximum number of parameters.
+bool MergeFuncIgnoringConstImpl::tryMapToParameter(FunctionInfos &FInfos,
+ unsigned OpIdx,
+ ParamInfos &Params,
+ unsigned maxParams) {
+ ParamInfo *Matching = nullptr;
+ // Try to find an existing parameter which exactly matches the differing
+ // operands of the current instruction.
+ for (ParamInfo &PI : Params) {
+ if (PI.matches(FInfos, OpIdx, isPtrAuthEnabled())) {
+ Matching = &PI;
+ break;
+ }
+ }
+ if (!Matching) {
+ // We need a new parameter.
+ // Check if we are within the limit.
+ if (Params.size() >= maxParams)
+ return false;
+
+ Params.resize(Params.size() + 1);
+ Matching = &Params.back();
+ // Store the constant values into the new parameter.
+ Constant *FirstC = cast<Constant>(FInfos[0].CurrentInst->getOperand(OpIdx));
+ for (FunctionInfo &FI : FInfos) {
+ Constant *C = cast<Constant>(FI.CurrentInst->getOperand(OpIdx));
+ Matching->Values.push_back(C);
+ if (C != FirstC)
+ FI.NumParamsNeeded += 1;
+ }
+ if (isPtrAuthEnabled())
+ Matching->needsPointerSigning = FInfos[0].needsPointerSigning(OpIdx);
+ }
+ /// Remember where the parameter is needed when we build our merged function.
+ Matching->Uses.push_back({FInfos[0].CurrentInst, OpIdx});
+ return true;
+}
+
+/// Copy \p origCall with a \p newCalle and add a ptrauth bundle with \p
+/// discriminator.
+void MergeFuncIgnoringConstImpl::replaceCallWithAddedPtrAuth(
+ CallInst *origCall, Value *newCallee, ConstantInt *discriminator) {
+ SmallVector<llvm::OperandBundleDef, 4> bundles;
+ origCall->getOperandBundlesAsDefs(bundles);
+ ConstantInt *key = getPtrAuthKey();
+ llvm::Value *bundleArgs[] = {key, discriminator};
+ bundles.emplace_back("ptrauth", bundleArgs);
+
+ SmallVector<llvm::Value *, 4> copiedArgs;
+ for (Value *op : origCall->args()) {
+ copiedArgs.push_back(op);
+ }
+
+ auto *newCall =
+ CallInst::Create(origCall->getFunctionType(), newCallee, copiedArgs,
+ bundles, origCall->getName(), origCall);
+ newCall->setAttributes(origCall->getAttributes());
+ newCall->setTailCallKind(origCall->getTailCallKind());
+ newCall->setCallingConv(origCall->getCallingConv());
+ origCall->replaceAllUsesWith(newCall);
+ origCall->eraseFromParent();
+}
+
+void MergeFuncIgnoringConstImpl::dumpMergeInfo(const FunctionInfos &FInfos,
+ unsigned paramSize) {
+ std::set<llvm::IRHash> oHashes;
+ std::vector<std::string> funcLocs;
+ Function *OrigFunc = nullptr;
+ for (const auto &FInfo : FInfos) {
+ OrigFunc = FInfo.F;
+
+ llvm::IRHash origHash = StructuralHash(*OrigFunc);
+ oHashes.insert(origHash);
+
+ // Print debug location.
+ std::string Result;
+ raw_string_ostream DbgLocOS(Result);
+ if (DISubprogram *DIS = OrigFunc->getSubprogram()) {
+ DebugLoc FuncDbgLoc =
+ DILocation::get(DIS->getContext(), DIS->getScopeLine(), 0, DIS);
+ FuncDbgLoc.print(DbgLocOS);
+ DbgLocOS.flush();
+ }
+ std::string singleLine =
+ "# functionLoc " +
+ std::to_string(GlobalValue::getGUID(OrigFunc->getName())) + " " +
+ Result + " " + std::string(OrigFunc->getName()) + "\n";
+ funcLocs.push_back(singleLine);
+ }
+}
+
+/// Merge all functions in \p FInfos by creating thunks which call the single
+/// merged function with additional parameters.
+void MergeFuncIgnoringConstImpl::mergeWithParams(const FunctionInfos &FInfos,
+ ParamInfos &Params) {
+ // We reuse the body of the first function for the new merged function.
+ Function *FirstF = FInfos.front().F;
+
+ // Build the type for the merged function. This will be the type of the
+ // original function (FirstF) but with the additional parameter which are
+ // needed to parameterize the merged function.
+ FunctionType *OrigTy = FirstF->getFunctionType();
+ SmallVector<Type *, 8> ParamTypes(OrigTy->param_begin(), OrigTy->param_end());
+
+ for (const ParamInfo &PI : Params) {
+ ParamTypes.push_back(PI.Values[0]->getType());
+ }
+
+ FunctionType *funcType =
+ FunctionType::get(OrigTy->getReturnType(), ParamTypes, false);
+
+ // Create the new function.
+ Function *NewFunction = Function::Create(funcType, FirstF->getLinkage(),
+ FirstF->getName() + ".Tm");
+ if (auto *SP = FirstF->getSubprogram())
+ NewFunction->setSubprogram(SP);
+ NewFunction->copyAttributesFrom(FirstF);
+ // NOTE: this function is not externally available, do ensure that we reset
+ // the DLL storage
+ NewFunction->setDLLStorageClass(GlobalValue::DefaultStorageClass);
+ if (UseLinkOnceODRLinkageMerging)
+ NewFunction->setLinkage(GlobalValue::LinkOnceODRLinkage);
+ else
+ NewFunction->setLinkage(GlobalValue::InternalLinkage);
+ if (NoInlineForMergedFunction)
+ NewFunction->addFnAttr(Attribute::NoInline);
+
+ // Insert the new function after the last function in the equivalence class.
+ FirstF->getParent()->getFunctionList().insert(
+ std::next(FInfos[1].F->getIterator()), NewFunction);
+
+ LLVM_DEBUG(dbgs() << " Merge into " << NewFunction->getName() << '\n');
+
+ // Move the body of FirstF into the NewFunction.
+ NewFunction->splice(NewFunction->begin(), FirstF);
+
+ auto NewArgIter = NewFunction->arg_begin();
+ for (Argument &OrigArg : FirstF->args()) {
+ Argument &NewArg = *NewArgIter++;
+ OrigArg.replaceAllUsesWith(&NewArg);
+ }
+ unsigned numOrigArgs = FirstF->arg_size();
+
+ SmallPtrSet<Function *, 8> SelfReferencingFunctions;
+
+ // Replace all differing operands with a parameter.
+ for (unsigned paramIdx = 0; paramIdx < Params.size(); ++paramIdx) {
+ const ParamInfo &PI = Params[paramIdx];
+ Argument *NewArg = NewFunction->getArg(numOrigArgs + paramIdx);
+
+ if (!PI.needsPointerSigning) {
+ for (const OpLocation &OL : PI.Uses) {
+ OL.I->setOperand(OL.OpIndex, NewArg);
+ }
+ }
+ // Collect all functions which are referenced by any parameter.
+ for (Value *V : PI.Values) {
+ if (auto *F = dyn_cast<Function>(V))
+ SelfReferencingFunctions.insert(F);
+ }
+ }
+
+ // Replace all differing operands, which need pointer signing, with a
+ // parameter.
+ // We need to do that after all other parameters, because here we replace
+ // call instructions, which must be live in case it has another constant to
+ // be replaced.
+ for (unsigned paramIdx = 0; paramIdx < Params.size(); ++paramIdx) {
+ ParamInfo &PI = Params[paramIdx];
+ if (PI.needsPointerSigning) {
+ PI.computeDiscriminator(NewFunction->getContext());
+ for (const OpLocation &OL : PI.Uses) {
+ auto *origCall = cast<CallInst>(OL.I);
+ Argument *newCallee = NewFunction->getArg(numOrigArgs + paramIdx);
+ replaceCallWithAddedPtrAuth(origCall, newCallee, PI.discriminator);
+ }
+ }
+ }
+
+ for (unsigned FIdx = 0, NumFuncs = FInfos.size(); FIdx < NumFuncs; ++FIdx) {
+ Function *OrigFunc = FInfos[FIdx].F;
+ // Don't try to replace all callers of functions which are used as
+ // parameters because we must not delete such functions.
+ if (SelfReferencingFunctions.count(OrigFunc) == 0 &&
+ replaceDirectCallers(OrigFunc, NewFunction, Params, FIdx)) {
+ // We could replace all uses (and the function is not externally visible),
+ // so we can delete the original function.
+ auto Iter = FuncEntries.find(OrigFunc);
+ assert(Iter != FuncEntries.end());
+ assert(!isInEquivalenceClass(&*Iter->second));
+ Iter->second->F = nullptr;
+ FuncEntries.erase(Iter);
+ LLVM_DEBUG(dbgs() << " Erase " << OrigFunc->getName() << '\n');
+ OrigFunc->eraseFromParent();
+ } else {
+ // Otherwise we need a thunk which calls the merged function.
+ writeThunk(NewFunction, OrigFunc, Params, FIdx);
+ }
+ ++NumFunctionsMergedIgnoringConst;
+ }
+}
+
+/// Remove all functions of \p FE's equivalence class from FnTree. Add them to
+/// Deferred so that we'll look at them in the next round.
+void MergeFuncIgnoringConstImpl::removeEquivalenceClassFromTree(
+ FunctionEntry *FE) {
+ if (!isInEquivalenceClass(FE))
+ return;
+
+ FnTreeType::iterator Iter = FE->TreeIter;
+ FunctionEntry *Unlink = Iter->First;
+ Unlink->numUnhandledCallees = 0;
+ while (Unlink) {
+ LLVM_DEBUG(dbgs() << " remove from tree: " << Unlink->F->getName()
+ << '\n');
+ if (!Unlink->isMerged)
+ Deferred.emplace_back(Unlink->F);
+ Unlink->TreeIter = FnTree.end();
+ assert(Unlink->numUnhandledCallees == 0);
+ FunctionEntry *NextEntry = Unlink->Next;
+ Unlink->Next = nullptr;
+ Unlink = NextEntry;
+ }
+ FnTree.erase(Iter);
+}
+
+// Helper for writeThunk,
+// Selects proper bitcast operation,
+// but a bit simpler then CastInst::getCastOpcode.
+Value *createCast(IRBuilder<> &Builder, Value *V, Type *DestTy) {
+ Type *SrcTy = V->getType();
+ if (SrcTy->isStructTy()) {
+ assert(DestTy->isStructTy());
+ assert(SrcTy->getStructNumElements() == DestTy->getStructNumElements());
+ Value *Result = UndefValue::get(DestTy);
+ for (unsigned int I = 0, E = SrcTy->getStructNumElements(); I < E; ++I) {
+ Value *Element =
+ createCast(Builder, Builder.CreateExtractValue(V, makeArrayRef(I)),
+ DestTy->getStructElementType(I));
+
+ Result = Builder.CreateInsertValue(Result, Element, makeArrayRef(I));
+ }
+ return Result;
+ }
+ assert(!DestTy->isStructTy());
+ if (CastArrayType) {
+ if (auto *SrcAT = dyn_cast<ArrayType>(SrcTy)) {
+ auto *DestAT = dyn_cast<ArrayType>(DestTy);
+ assert(DestAT);
+ assert(SrcAT->getNumElements() == DestAT->getNumElements());
+ Value *Result = UndefValue::get(DestTy);
+ for (unsigned int I = 0, E = SrcAT->getNumElements(); I < E; ++I) {
+ Value *Element =
+ createCast(Builder, Builder.CreateExtractValue(V, makeArrayRef(I)),
+ DestAT->getElementType());
+
+ Result = Builder.CreateInsertValue(Result, Element, makeArrayRef(I));
+ }
+ return Result;
+ }
+ assert(!DestTy->isArrayTy());
+ }
+ if (SrcTy->isIntegerTy() && DestTy->isPointerTy())
+ return Builder.CreateIntToPtr(V, DestTy);
+ else if (SrcTy->isPointerTy() && DestTy->isIntegerTy())
+ return Builder.CreatePtrToInt(V, DestTy);
+ else
+ return Builder.CreateBitCast(V, DestTy);
+}
+
+/// Replace \p Thunk with a simple tail call to \p ToFunc. Also add parameters
+/// to the call to \p ToFunc, which are defined by the FuncIdx's value in
+/// \p Params.
+void MergeFuncIgnoringConstImpl::writeThunk(Function *ToFunc, Function *Thunk,
+ const ParamInfos &Params,
+ unsigned FuncIdx) {
+ // Delete the existing content of Thunk.
+ Thunk->dropAllReferences();
+
+ BasicBlock *BB = BasicBlock::Create(Thunk->getContext(), "", Thunk);
+ IRBuilder<> Builder(BB);
+
+ SmallVector<Value *, 16> Args;
+ unsigned ParamIdx = 0;
+ FunctionType *ToFuncTy = ToFunc->getFunctionType();
+
+ // Add arguments which are passed through Thunk.
+ for (Argument &AI : Thunk->args()) {
+ Args.push_back(createCast(Builder, &AI, ToFuncTy->getParamType(ParamIdx)));
+ ++ParamIdx;
+ }
+ // Add new arguments defined by Params.
+ for (const ParamInfo &PI : Params) {
+ assert(ParamIdx < ToFuncTy->getNumParams());
+ Constant *param = getSignedValue(PI, FuncIdx);
+ Args.push_back(
+ createCast(Builder, param, ToFuncTy->getParamType(ParamIdx)));
+ ++ParamIdx;
+ }
+
+ CallInst *CI = Builder.CreateCall(ToFunc, Args);
+ bool isSwiftTailCall = ToFunc->getCallingConv() == CallingConv::SwiftTail &&
+ Thunk->getCallingConv() == CallingConv::SwiftTail;
+ CI->setTailCallKind(isSwiftTailCall ? llvm::CallInst::TCK_MustTail
+ : llvm::CallInst::TCK_Tail);
+ CI->setCallingConv(ToFunc->getCallingConv());
+ CI->setAttributes(ToFunc->getAttributes());
+ if (Thunk->getReturnType()->isVoidTy()) {
+ Builder.CreateRetVoid();
+ } else {
+ Builder.CreateRet(createCast(Builder, CI, Thunk->getReturnType()));
+ }
+
+ LLVM_DEBUG(dbgs() << " writeThunk: " << Thunk->getName() << '\n');
+ ++NumThunksWrittenIgnoringConst;
+}
+
+/// Replace direct callers of Old with New. Also add parameters to the call to
+/// \p New, which are defined by the FuncIdx's value in \p Params.
+bool MergeFuncIgnoringConstImpl::replaceDirectCallers(Function *Old,
+ Function *New,
+ const ParamInfos &Params,
+ unsigned FuncIdx) {
+ bool AllReplaced = true;
+
+ SmallVector<CallInst *, 8> Callers;
+
+ for (Use &U : Old->uses()) {
+ auto *I = dyn_cast<Instruction>(U.getUser());
+ if (!I) {
+ AllReplaced = false;
+ continue;
+ }
+ FunctionEntry *FE = getEntry(I->getFunction());
+ if (FE)
+ removeEquivalenceClassFromTree(FE);
+
+ auto *CI = dyn_cast<CallInst>(I);
+ if (!CI || CI->getCalledOperand() != Old) {
+ AllReplaced = false;
+ continue;
+ }
+ Callers.push_back(CI);
+ }
+ if (!AllReplaced)
+ return false;
+
+ // When AlwaysCallThunk is true, return false so a thunk will be emitted, also
+ // do not replace callsites.
+ if (AlwaysCallThunk)
+ return false;
+
+ for (CallInst *CI : Callers) {
+ auto &Context = New->getContext();
+ auto NewPAL = New->getAttributes();
+
+ SmallVector<Type *, 8> OldParamTypes;
+ SmallVector<Value *, 16> NewArgs;
+ SmallVector<AttributeSet, 8> NewArgAttrs;
+ IRBuilder<> Builder(CI);
+
+ FunctionType *NewFuncTy = New->getFunctionType();
+ (void)NewFuncTy;
+ unsigned ParamIdx = 0;
+
+ // Add the existing parameters.
+ for (Value *OldArg : CI->args()) {
+ NewArgAttrs.push_back(NewPAL.getParamAttrs(ParamIdx));
+ NewArgs.push_back(OldArg);
+ OldParamTypes.push_back(OldArg->getType());
+ ++ParamIdx;
+ }
+ // Add the new parameters.
+ for (const ParamInfo &PI : Params) {
+ assert(ParamIdx < NewFuncTy->getNumParams());
+ Constant *ArgValue = getSignedValue(PI, FuncIdx);
+ assert(ArgValue != Old && "should not try to replace all callers of self "
+ "referencing functions");
+ NewArgs.push_back(ArgValue);
+ OldParamTypes.push_back(ArgValue->getType());
+ ++ParamIdx;
+ }
+
+ auto *FType = FunctionType::get(Old->getFunctionType()->getReturnType(),
+ OldParamTypes, false);
+ auto *FPtrType = PointerType::get(
+ FType, cast<PointerType>(New->getType())->getAddressSpace());
+
+ Value *Callee = ConstantExpr::getBitCast(New, FPtrType);
+ CallInst *NewCI;
+ if (objcarc::hasAttachedCallOpBundle(CI)) {
+ Value *BundleArgs[] = {*objcarc::getAttachedARCFunction(CI)};
+ OperandBundleDef OB("clang.arc.attachedcall", BundleArgs);
+ NewCI = Builder.CreateCall(FType, Callee, NewArgs, {OB});
+ } else {
+ NewCI = Builder.CreateCall(FType, Callee, NewArgs);
+ }
+ NewCI->setCallingConv(CI->getCallingConv());
+ // Don't transfer attributes from the function to the callee. Function
+ // attributes typically aren't relevant to the calling convention or ABI.
+ NewCI->setAttributes(AttributeList::get(Context, /*FnAttrs=*/AttributeSet(),
+ NewPAL.getRetAttrs(), NewArgAttrs));
+ if (IgnoreMusttailFunction && CI->isMustTailCall()) {
+ // replace a callsite with musttail.
+ llvm::errs() << "callsite has musttail in newF " << New->getName()
+ << "\n";
+ }
+ NewCI->copyMetadata(*CI);
+ CI->replaceAllUsesWith(NewCI);
+ CI->eraseFromParent();
+ }
+ assert(Old->use_empty() && "should have replaced all uses of old function");
+ return Old->hasLocalLinkage();
+}
+
+PreservedAnalyses MergeFuncIgnoringConstPass::run(Module &M,
+ ModuleAnalysisManager &MAM) {
+ if (runInternal(M))
+ return PreservedAnalyses::none();
+ return PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index e971c638327bf05..80946cb06547551 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -27,6 +27,8 @@ add_llvm_component_library(LLVMTransformUtils
FixIrreducible.cpp
FlattenCFG.cpp
FunctionComparator.cpp
+ FunctionComparatorIgnoringConst.cpp
+ FunctionHashIgnoringConst.cpp
FunctionImportUtils.cpp
GlobalStatus.cpp
GuardUtils.cpp
diff --git a/llvm/lib/Transforms/Utils/FunctionComparatorIgnoringConst.cpp b/llvm/lib/Transforms/Utils/FunctionComparatorIgnoringConst.cpp
new file mode 100644
index 000000000000000..3b3567111f43034
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/FunctionComparatorIgnoringConst.cpp
@@ -0,0 +1,107 @@
+//===--- FunctionComparatorIgnoringConst.cpp - Function Comparator --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/FunctionComparatorIgnoringConst.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/Transforms/Utils/MergeFunctionsIgnoringConst.h"
+
+using namespace llvm;
+
+int FunctionComparatorIgnoringConst::cmpOperandsIgnoringConsts(
+ const Instruction *L, const Instruction *R, unsigned opIdx) {
+ Value *OpL = L->getOperand(opIdx);
+ Value *OpR = R->getOperand(opIdx);
+
+ int Res = cmpValues(OpL, OpR);
+ if (Res == 0)
+ return Res;
+
+ if (!isa<Constant>(OpL) || !isa<Constant>(OpR))
+ return Res;
+
+ if (!isEligibleOperandForConstantSharing(L, opIdx) ||
+ !isEligibleOperandForConstantSharing(R, opIdx))
+ return Res;
+
+ if (cmpTypes(OpL->getType(), OpR->getType()))
+ return Res;
+
+ return 0;
+}
+
+// Test whether two basic blocks have equivalent behavior.
+int FunctionComparatorIgnoringConst::cmpBasicBlocksIgnoringConsts(
+ const BasicBlock *BBL, const BasicBlock *BBR,
+ const std::set<std::pair<int, int>> *InstOpndIndex) {
+ BasicBlock::const_iterator InstL = BBL->begin(), InstLE = BBL->end();
+ BasicBlock::const_iterator InstR = BBR->begin(), InstRE = BBR->end();
+
+ do {
+ bool needToCmpOperands = true;
+ if (int Res = cmpOperations(&*InstL, &*InstR, needToCmpOperands))
+ return Res;
+ if (needToCmpOperands) {
+ assert(InstL->getNumOperands() == InstR->getNumOperands());
+
+ for (unsigned i = 0, e = InstL->getNumOperands(); i != e; ++i) {
+ // When a set for (instruction, operand) index pairs is given, we only
+ // ignore constants located at such indices. Otherwise, we precisely
+ // compare the operands.
+ if (InstOpndIndex && !InstOpndIndex->count(std::make_pair(index, i))) {
+ Value *OpL = InstL->getOperand(i);
+ Value *OpR = InstR->getOperand(i);
+ if (int Res = cmpValues(OpL, OpR))
+ return Res;
+ }
+ if (int Res = cmpOperandsIgnoringConsts(&*InstL, &*InstR, i))
+ return Res;
+ // cmpValues should ensure this is true.
+ assert(cmpTypes(InstL->getOperand(i)->getType(),
+ InstR->getOperand(i)->getType()) == 0);
+ }
+ }
+ ++index;
+ ++InstL, ++InstR;
+ } while (InstL != InstLE && InstR != InstRE);
+
+ if (InstL != InstLE && InstR == InstRE)
+ return 1;
+ if (InstL == InstLE && InstR != InstRE)
+ return -1;
+ return 0;
+}
+
+// Test whether the two functions have equivalent behavior.
+int FunctionComparatorIgnoringConst::compareIgnoringConsts(
+ const std::set<std::pair<int, int>> *InstOpndIndex) {
+ beginCompare();
+ index = 0;
+
+ if (int Res = compareSignature())
+ return Res;
+
+ Function::const_iterator LIter = FnL->begin(), LEnd = FnL->end();
+ Function::const_iterator RIter = FnR->begin(), REnd = FnR->end();
+
+ do {
+ const BasicBlock *BBL = &*LIter;
+ const BasicBlock *BBR = &*RIter;
+
+ if (int Res = cmpValues(BBL, BBR))
+ return Res;
+
+ if (int Res = cmpBasicBlocksIgnoringConsts(BBL, BBR, InstOpndIndex))
+ return Res;
+
+ ++LIter, ++RIter;
+ } while (LIter != LEnd && RIter != REnd);
+
+ return 0;
+}
diff --git a/llvm/lib/Transforms/Utils/FunctionHashIgnoringConst.cpp b/llvm/lib/Transforms/Utils/FunctionHashIgnoringConst.cpp
new file mode 100644
index 000000000000000..b24d3ffdba93388
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/FunctionHashIgnoringConst.cpp
@@ -0,0 +1,620 @@
+//===--- FunctionHashIgnoringConst.cpp - Function Hash --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/FunctionHashIgnoringConst.h"
+#include "llvm/CodeGen/MachineStableHash.h"
+#include "llvm/IR/InlineAsm.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Transforms/Utils/MergeFunctionsIgnoringConst.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "functionhash-ignoringconst"
+
+namespace {
+
+// Accumulate the hash of a sequence of 64-bit integers. This is similar to a
+// hash of a sequence of 64bit ints, but the entire input does not need to be
+// available at once. This interface is necessary for functionHash because it
+// needs to accumulate the hash as the structure of the function is traversed
+// without saving these values to an intermediate buffer. This form of hashing
+// is not often needed, as usually the object to hash is just read from a
+// buffer.
+class HashAccumulator64 {
+ uint64_t Hash;
+
+public:
+ // Initialize to random constant, so the state isn't zero.
+ HashAccumulator64() { Hash = 0x6acaa36bef8325c5ULL; }
+
+ void add(uint64_t V) { Hash = hashing::detail::hash_16_bytes(Hash, V); }
+
+ // No finishing is required, because the entire hash value is used.
+ uint64_t getHash() { return Hash; }
+};
+
+} // end anonymous namespace
+
+FunctionHashIgnoringConst::FunctionHash FunctionHashIgnoringConst::functionHash(
+ Function &F, std::map<int, Instruction *> *IdxToInst,
+ std::map<std::pair<int, int>, uint64_t> *IdxToConstHash) {
+ GlobalNumberState tGlobalNumbers;
+ FunctionHashIgnoringConst pFC(&F, &tGlobalNumbers);
+
+ std::map<std::pair<int, int>, uint64_t> LIdxToConstHash;
+ std::map<int, Instruction *> LIdxToIns;
+ if (!IdxToConstHash)
+ IdxToConstHash = &LIdxToConstHash;
+ if (!IdxToInst)
+ IdxToInst = &LIdxToIns;
+
+ auto Hash = pFC.hashIgnoringConsts(*IdxToInst, *IdxToConstHash);
+ // FIXME: Check if all Constants with the same hash are identical.
+ // Conservatively, we return 0 hash if that's not the case.
+ // Should we just assert it?
+ if (Hash && IdxToConstHash && IdxToInst) {
+ std::map<uint64_t, Constant *> ConstHashToConst;
+ for (auto &P : *IdxToConstHash) {
+ auto InstIndex = P.first.first;
+ auto OpndIndex = P.first.second;
+ auto ConstHash = P.second;
+ auto *Inst = (*IdxToInst)[InstIndex];
+ auto *Const = cast<Constant>(Inst->getOperand(OpndIndex));
+ auto It = ConstHashToConst.find(ConstHash);
+ if (It != ConstHashToConst.end()) {
+ auto *PrevConst = It->second;
+ if (pFC.cmpConstants(PrevConst, Const))
+ return 0;
+ } else {
+ ConstHashToConst.insert({ConstHash, Const});
+ }
+ }
+ }
+ return Hash;
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashIgnoringConsts(
+ std::map<int, Instruction *> &IdxToInst,
+ std::map<std::pair<int, int>, uint64_t> &IdxToConstHash) {
+ beginCompare();
+ index = 0;
+
+ auto h = hashSignature();
+ Function::const_iterator LIter = FnL->begin(), LEnd = FnL->end();
+
+ do {
+ const BasicBlock *BBL = &*LIter;
+ auto h1 = hashValue(BBL);
+ auto h2 = hashBasicBlocksIgnoringConsts(BBL, IdxToInst, IdxToConstHash);
+ // Ignore 0 hash value conservatively.
+ if (h2 == 0)
+ return 0;
+
+ h = hash_combine(h, h1, h2);
+ ++LIter;
+ } while (LIter != LEnd);
+ return h;
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashBasicBlocksIgnoringConsts(
+ const BasicBlock *BBL, std::map<int, Instruction *> &IdxToInst,
+ std::map<std::pair<int, int>, uint64_t> &IdxToConstHash) {
+
+ BasicBlock::const_iterator InstL = BBL->begin(), InstLE = BBL->end();
+ bool isFirst = true;
+ FunctionHashIgnoringConst::FunctionHash retH;
+ do {
+ bool needToCmpOperands = true;
+ auto h = hashOperation(&*InstL, needToCmpOperands);
+ if (needToCmpOperands) {
+ for (int i = 0, e = InstL->getNumOperands(); i != e; ++i) {
+ auto hashOpnd = 0;
+ Value *OpL = InstL->getOperand(i);
+ if (isa<Constant>(OpL)) {
+ hashOpnd = hashConstant(cast<Constant>(OpL));
+ if (isEligibleOperandForConstantSharing(&*InstL, i)) {
+ // Preserve the original const hash in the map while
+ // ignoring it while hashing its type only.
+ IdxToConstHash[std::make_pair(index, i)] = hashOpnd;
+ hashOpnd = hashType(OpL->getType());
+ }
+ } else {
+ hashOpnd = hashValue(OpL);
+ }
+ h = hash_combine(h, hashOpnd);
+ }
+ }
+ IdxToInst[index] = (Instruction *)&*InstL;
+ ++index;
+ if (isFirst) {
+ retH = h;
+ isFirst = false;
+ } else
+ retH = hash_combine(retH, h);
+ ++InstL;
+ } while (InstL != InstLE);
+ return retH;
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashAPInt(const APInt &L) const {
+ HashAccumulator64 H;
+ H.add(hash_value(L));
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashAPFloat(const APFloat &L) const {
+ const fltSemantics &SL = L.getSemantics();
+ HashAccumulator64 H;
+ H.add(APFloat::semanticsPrecision(SL));
+ H.add(APFloat::semanticsMaxExponent(SL));
+ H.add(APFloat::semanticsMinExponent(SL));
+ H.add(APFloat::semanticsSizeInBits(SL));
+ auto h = hashAPInt(L.bitcastToAPInt());
+ H.add(h);
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashAttrs(const AttributeList L) const {
+ HashAccumulator64 H;
+ H.add(L.getNumAttrSets());
+
+ for (unsigned i : L.indexes()) {
+ AttributeSet LAS = L.getAttributes(i);
+ AttributeSet::iterator LI = LAS.begin(), LE = LAS.end();
+ for (; LI != LE; ++LI) {
+ Attribute LA = *LI;
+ H.add(LA.isTypeAttribute());
+ if (LA.isTypeAttribute()) {
+ H.add(LA.getKindAsEnum());
+ Type *TyL = LA.getValueAsType();
+ if (TyL) {
+ auto h = hashType(TyL);
+ H.add(h);
+ }
+ continue;
+ }
+ // Check AttributeImpl::operator<
+ if (!LA.isStringAttribute()) {
+ H.add(LA.getKindAsEnum());
+ if (LA.isIntAttribute())
+ H.add(LA.getValueAsInt());
+ continue;
+ }
+ H.add(stable_hash_combine_string(LA.getKindAsString()));
+ H.add(stable_hash_combine_string(LA.getValueAsString()));
+ }
+ }
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashRangeMetadata(const MDNode *L) const {
+ HashAccumulator64 H;
+ if (!L) {
+ H.add('N');
+ return H.getHash();
+ }
+ H.add(L->getNumOperands());
+ for (size_t I = 0; I < L->getNumOperands(); ++I) {
+ ConstantInt *LLow = mdconst::extract<ConstantInt>(L->getOperand(I));
+ H.add(hashAPInt(LLow->getValue()));
+ }
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashOperandBundlesSchema(const CallBase &LCS) const {
+ HashAccumulator64 H;
+ H.add(LCS.getNumOperandBundles());
+ for (unsigned I = 0, E = LCS.getNumOperandBundles(); I != E; ++I) {
+ auto OBL = LCS.getOperandBundleAt(I);
+ H.add(stable_hash_combine_string(OBL.getTagName()));
+ H.add(OBL.Inputs.size());
+ }
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashConstant(const Constant *L) const {
+ HashAccumulator64 H;
+
+ Type *TyL = L->getType();
+ // if (!TyL->isfirstClassType())
+ // For first class type, check if can be bitcasted
+ unsigned TyLWidth = 0;
+ if (auto *VecTyL = dyn_cast<VectorType>(TyL)) {
+ H.add(stable_hash_combine_string("V"));
+ TyLWidth = VecTyL->getPrimitiveSizeInBits().getFixedSize();
+ H.add(TyLWidth);
+ } else if (auto *PTyL = dyn_cast<PointerType>(TyL)) {
+ H.add(stable_hash_combine_string("P"));
+ H.add(PTyL->getAddressSpace());
+ } else {
+ auto h = hashType(TyL);
+ H.add(h);
+ }
+ if (L->isNullValue()) {
+ H.add(stable_hash_combine_string("N"));
+ return H.getHash();
+ }
+ auto GlobalValueL = const_cast<GlobalValue *>(dyn_cast<GlobalValue>(L));
+ if (GlobalValueL) {
+ auto h = hashGlobalValue(GlobalValueL);
+ // Ignore 0 hash value conservatively.
+ if (h == 0)
+ return 0;
+ H.add(h);
+ return H.getHash();
+ }
+ if (const auto *SeqL = dyn_cast<ConstantDataSequential>(L)) {
+ H.add(stable_hash_combine_string(SeqL->getRawDataValues()));
+ return H.getHash();
+ }
+
+ // This is right no hashing getValueID() like comparing it in cmpConstants
+ // since the id itself can vary.
+ switch (L->getValueID()) {
+ case Value::UndefValueVal:
+ case Value::PoisonValueVal:
+ case Value::ConstantTokenNoneVal: {
+ return H.getHash();
+ }
+ case Value::ConstantIntVal: {
+ const APInt &LInt = cast<ConstantInt>(L)->getValue();
+ H.add(hashAPInt(LInt));
+ return H.getHash();
+ }
+ case Value::ConstantFPVal: {
+ const APFloat &LAPF = cast<ConstantFP>(L)->getValueAPF();
+ H.add(hashAPFloat(LAPF));
+ return H.getHash();
+ }
+ case Value::ConstantArrayVal: {
+ const ConstantArray *LA = cast<ConstantArray>(L);
+ uint64_t NumElementsL = cast<ArrayType>(TyL)->getNumElements();
+ H.add(NumElementsL);
+ for (uint64_t i = 0; i < NumElementsL; ++i) {
+ auto h = hashConstant(cast<Constant>(LA->getOperand(i)));
+ H.add(h);
+ }
+ return H.getHash();
+ }
+ case Value::ConstantStructVal: {
+ const ConstantStruct *LS = cast<ConstantStruct>(L);
+ unsigned NumElementsL = cast<StructType>(TyL)->getNumElements();
+ H.add(NumElementsL);
+ for (unsigned i = 0; i != NumElementsL; ++i) {
+ auto h = hashConstant(cast<Constant>(LS->getOperand(i)));
+ H.add(h);
+ }
+ return H.getHash();
+ }
+ case Value::ConstantVectorVal: {
+ const ConstantVector *LV = cast<ConstantVector>(L);
+ unsigned NumElementsL = cast<FixedVectorType>(TyL)->getNumElements();
+ H.add(NumElementsL);
+ for (uint64_t i = 0; i < NumElementsL; ++i) {
+ auto h = hashConstant(cast<Constant>(LV->getOperand(i)));
+ H.add(h);
+ }
+ return H.getHash();
+ }
+ case Value::ConstantExprVal: {
+ const ConstantExpr *LE = cast<ConstantExpr>(L);
+ unsigned NumOperandsL = LE->getNumOperands();
+ H.add(NumOperandsL);
+ for (unsigned i = 0; i < NumOperandsL; ++i) {
+ auto h = hashConstant(cast<Constant>(LE->getOperand(i)));
+ H.add(h);
+ }
+ return H.getHash();
+ }
+ case Value::BlockAddressVal: {
+ const BlockAddress *LBA = cast<BlockAddress>(L);
+ auto h = hashGlobalValue(LBA->getFunction());
+ // TODO: handle BBs in the same function. can we reference a block
+ // in another TU?
+ H.add(h);
+ return H.getHash();
+ }
+ default: // Unknown constant, abort.
+ LLVM_DEBUG(dbgs() << "Looking at valueID " << L->getValueID() << "\n");
+ llvm_unreachable("Constant ValueID not recognized.");
+ }
+
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashGlobalValue(const GlobalValue *GV) const {
+ HashAccumulator64 H;
+
+ if (!GV->hasName())
+ return 0;
+
+ // For the local global, has module identifier to make it unique.
+ if (GV->hasLocalLinkage()) {
+ H.add(stable_hash_combine_string(GV->getParent()->getModuleIdentifier()));
+ }
+
+ // Use GUID to hash more consistently.
+ H.add(GV->getGUID());
+
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashType(Type *TyL) const {
+ HashAccumulator64 H;
+ PointerType *PTyL = dyn_cast<PointerType>(TyL);
+ const DataLayout &DL = FnL->getParent()->getDataLayout();
+ if (PTyL && PTyL->getAddressSpace() == 0)
+ TyL = DL.getIntPtrType(TyL);
+ H.add(TyL->getTypeID());
+ switch (TyL->getTypeID()) {
+ default:
+ break;
+ case Type::IntegerTyID:
+ H.add(cast<IntegerType>(TyL)->getBitWidth());
+ break;
+ case Type::PointerTyID:
+ H.add(PTyL->getAddressSpace());
+ break;
+ case Type::StructTyID: {
+ StructType *STyL = cast<StructType>(TyL);
+ H.add(STyL->getNumElements());
+ H.add(STyL->isPacked());
+ for (unsigned i = 0, e = STyL->getNumElements(); i != e; ++i) {
+ auto h = hashType(STyL->getElementType(i));
+ H.add(h);
+ }
+ break;
+ }
+ case Type::FunctionTyID: {
+ FunctionType *FTyL = cast<FunctionType>(TyL);
+ H.add(FTyL->getNumParams());
+ H.add(FTyL->isVarArg());
+ auto h = hashType(FTyL->getReturnType());
+ H.add(h);
+
+ for (unsigned i = 0, e = FTyL->getNumParams(); i != e; ++i) {
+ auto h = hashType(FTyL->getParamType(i));
+ H.add(h);
+ }
+ break;
+ }
+ case Type::ArrayTyID: {
+ auto *STyL = cast<ArrayType>(TyL);
+ H.add(STyL->getNumElements());
+ auto h = hashType(STyL->getElementType());
+ H.add(h);
+ break;
+ }
+ case Type::FixedVectorTyID:
+ case Type::ScalableVectorTyID: {
+ auto *STyL = cast<VectorType>(TyL);
+ H.add(STyL->getElementCount().isScalable());
+ H.add(STyL->getElementCount().getKnownMinValue());
+ auto h = hashType(STyL->getElementType());
+ H.add(h);
+ break;
+ }
+ }
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashOperation(const Instruction *L,
+ bool &needToCmpOperands) const {
+ needToCmpOperands = true;
+ HashAccumulator64 H;
+ auto h = hashValue(L);
+ H.add(h);
+ H.add(L->getOpcode());
+ if (const GetElementPtrInst *GEPL = dyn_cast<GetElementPtrInst>(L)) {
+ needToCmpOperands = false;
+ auto h = hashValue(GEPL->getPointerOperand());
+ H.add(h);
+ h = hashGEP(cast<GEPOperator>(GEPL));
+ H.add(h);
+ return H.getHash();
+ }
+ H.add(L->getNumOperands());
+ H.add(hashType(L->getType()));
+ H.add(L->getRawSubclassOptionalData());
+
+ for (unsigned i = 0, e = L->getNumOperands(); i != e; ++i) {
+ H.add(hashType(L->getOperand(i)->getType()));
+ }
+
+ if (const AllocaInst *AI = dyn_cast<AllocaInst>(L)) {
+ H.add(hashType(AI->getAllocatedType()));
+ H.add(AI->getAlign().value());
+ return H.getHash();
+ }
+ if (const LoadInst *LI = dyn_cast<LoadInst>(L)) {
+ H.add(LI->isVolatile());
+ H.add(LI->getAlign().value());
+ H.add((int)(LI->getOrdering()));
+ H.add(LI->getSyncScopeID());
+ H.add(hashRangeMetadata(LI->getMetadata(LLVMContext::MD_range)));
+ return H.getHash();
+ }
+ if (const StoreInst *SI = dyn_cast<StoreInst>(L)) {
+ H.add(SI->isVolatile());
+ H.add(SI->getAlign().value());
+ H.add((int)(SI->getOrdering()));
+ H.add(SI->getSyncScopeID());
+ return H.getHash();
+ }
+ if (const CmpInst *CI = dyn_cast<CmpInst>(L)) {
+ H.add(CI->getPredicate());
+ return H.getHash();
+ }
+ if (auto *CBL = dyn_cast<CallBase>(L)) {
+ H.add(CBL->getCallingConv());
+ H.add(hashAttrs(CBL->getAttributes()));
+ H.add(hashOperandBundlesSchema(*CBL));
+ if (const CallInst *CI = dyn_cast<CallInst>(L))
+ H.add(CI->getTailCallKind());
+ H.add(hashRangeMetadata(L->getMetadata(LLVMContext::MD_range)));
+ return H.getHash();
+ }
+ if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(L)) {
+ ArrayRef<unsigned> LIndices = IVI->getIndices();
+ H.add(LIndices.size());
+ for (size_t i = 0, e = LIndices.size(); i != e; ++i) {
+ H.add(LIndices[i]);
+ }
+ return H.getHash();
+ }
+ if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(L)) {
+ ArrayRef<unsigned> LIndices = EVI->getIndices();
+ H.add(LIndices.size());
+ for (size_t i = 0, e = LIndices.size(); i != e; ++i) {
+ H.add(LIndices[i]);
+ }
+ return H.getHash();
+ }
+ if (const FenceInst *FI = dyn_cast<FenceInst>(L)) {
+ H.add((int)(FI->getOrdering()));
+ H.add(FI->getSyncScopeID());
+ return H.getHash();
+ }
+ if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(L)) {
+ H.add(CXI->isVolatile());
+ H.add(CXI->isWeak());
+ H.add((int)(CXI->getSuccessOrdering()));
+ H.add((int)(CXI->getFailureOrdering()));
+ H.add(CXI->getSyncScopeID());
+ return H.getHash();
+ }
+ if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(L)) {
+ H.add(RMWI->getOperation());
+ H.add(RMWI->isVolatile());
+ H.add((int)(RMWI->getOrdering()));
+ H.add(RMWI->getSyncScopeID());
+ return H.getHash();
+ }
+ if (const ShuffleVectorInst *SVI = dyn_cast<ShuffleVectorInst>(L)) {
+ ArrayRef<int> LMask = SVI->getShuffleMask();
+ H.add(LMask.size());
+ for (size_t i = 0, e = LMask.size(); i != e; ++i) {
+ H.add(LMask[i]);
+ }
+ return H.getHash();
+ }
+ if (const PHINode *PNL = dyn_cast<PHINode>(L)) {
+ for (unsigned i = 0, e = PNL->getNumIncomingValues(); i != e; ++i) {
+ H.add(hashValue(PNL->getIncomingBlock(i)));
+ }
+ return H.getHash();
+ }
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashGEP(const GEPOperator *GEPL) const {
+ unsigned int ASL = GEPL->getPointerAddressSpace();
+ HashAccumulator64 H;
+ H.add(ASL);
+
+ const DataLayout &DL = FnL->getParent()->getDataLayout();
+ unsigned BitWidth = DL.getPointerSizeInBits(ASL);
+ APInt OffsetL(BitWidth, 0);
+ if (GEPL->accumulateConstantOffset(DL, OffsetL)) {
+ H.add(hashAPInt(OffsetL));
+ // Return early similar to how we implement cmpGEPs.
+ // TODO: Should we let it got thru the followings?
+ return H.getHash();
+ }
+
+ H.add(hashType(GEPL->getSourceElementType()));
+ H.add(GEPL->getNumOperands());
+ for (unsigned i = 0, e = GEPL->getNumOperands(); i != e; ++i) {
+ H.add(hashValue(GEPL->getOperand(i)));
+ }
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashInlineAsm(const InlineAsm *L) const {
+ HashAccumulator64 H;
+ auto h = hashType(L->getFunctionType());
+ H.add(h);
+ H.add(stable_hash_combine_string(L->getAsmString()));
+ H.add(stable_hash_combine_string(L->getConstraintString()));
+ H.add(L->hasSideEffects());
+ H.add(L->isAlignStack());
+ H.add(L->getDialect());
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashValue(const Value *L) const {
+ HashAccumulator64 H;
+ if (L == FnL) {
+ H.add(stable_hash_combine_string("S"));
+ return H.getHash();
+ }
+
+ const Constant *ConstL = dyn_cast<Constant>(L);
+ if (ConstL) {
+ H.add(stable_hash_combine_string("C"));
+ auto h = hashConstant(ConstL);
+ H.add(h);
+ return H.getHash();
+ }
+
+ const InlineAsm *InlineAsmL = dyn_cast<InlineAsm>(L);
+ if (InlineAsmL)
+ return hashInlineAsm(InlineAsmL);
+
+ // TODO: hash the index at the first insertion to the map? what if we don't
+ // have a map???
+ auto LeftSN = sn_mapL.insert(std::make_pair(L, sn_mapL.size()));
+ H.add(LeftSN.first->second);
+ return H.getHash();
+}
+
+FunctionHashIgnoringConst::FunctionHash
+FunctionHashIgnoringConst::hashSignature() const {
+ HashAccumulator64 H;
+ auto h = hashAttrs(FnL->getAttributes());
+ H.add(h);
+ H.add(FnL->hasGC());
+ if (FnL->hasGC()) {
+ uint64_t rawHash = stable_hash_combine_string(FnL->getGC());
+ H.add(rawHash);
+ }
+ H.add(FnL->hasSection());
+ if (FnL->hasSection()) {
+ uint64_t rawHash = stable_hash_combine_string(FnL->getSection());
+ H.add(rawHash);
+ }
+ H.add(FnL->isVarArg());
+ H.add(FnL->getCallingConv());
+ h = hashType(FnL->getFunctionType());
+ H.add(h);
+ for (Function::const_arg_iterator ArgI = FnL->arg_begin(),
+ ArgE = FnL->arg_end();
+ ArgI != ArgE; ++ArgI) {
+ h = hashValue(&*ArgI);
+ H.add(h);
+ }
+ return H.getHash();
+}
diff --git a/llvm/unittests/Transforms/Utils/CMakeLists.txt b/llvm/unittests/Transforms/Utils/CMakeLists.txt
index d1714a7532d5ff4..a40ec222a0bfb56 100644
--- a/llvm/unittests/Transforms/Utils/CMakeLists.txt
+++ b/llvm/unittests/Transforms/Utils/CMakeLists.txt
@@ -19,6 +19,7 @@ add_llvm_unittest(UtilsTests
CodeMoverUtilsTest.cpp
DebugifyTest.cpp
FunctionComparatorTest.cpp
+ FunctionHashIgnoringConstTest.cpp
IntegerDivisionTest.cpp
LocalTest.cpp
LoopRotationUtilsTest.cpp
diff --git a/llvm/unittests/Transforms/Utils/FunctionHashIgnoringConstTest.cpp b/llvm/unittests/Transforms/Utils/FunctionHashIgnoringConstTest.cpp
new file mode 100644
index 000000000000000..64e7b96170979e6
--- /dev/null
+++ b/llvm/unittests/Transforms/Utils/FunctionHashIgnoringConstTest.cpp
@@ -0,0 +1,120 @@
+//===---- FunctionHashIgnoringConstTest.cpp - Unit tests -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/FunctionHashIgnoringConst.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+static std::unique_ptr<Module> parseIR(LLVMContext &C, const char *IR) {
+ SMDiagnostic Err;
+ std::unique_ptr<Module> Mod = parseAssemblyString(IR, Err, C);
+ if (!Mod)
+ Err.print("ModuleUtilsTest", errs());
+ return Mod;
+}
+
+// FuncionHashIgnoringConst ignores some Constant using
+// isEligibleInstrunctionForConstantSharing().
+// The following tests check those globals operated by load/store/call.
+TEST(FunctionHashIgnoringConst, HashPass) {
+ LLVMContext C;
+ const char *ModStr = "@g1 = external global i32\n"
+ "@g2 = external global i32\n"
+ "declare void @f1()\n"
+ "declare void @f2()\n"
+ "define i32 @foo() {\n"
+ " %1 = load i32, i32* @g1\n"
+ " store i32 %1, i32* @g1\n"
+ " call void @f1()"
+ " ret i32 %1\n"
+ "}\n"
+ "define i32 @goo() {\n"
+ " %1 = load i32, i32* @g2\n"
+ " store i32 %1, i32* @g2\n"
+ " call void @f2()"
+ " ret i32 %1\n"
+ "}\n";
+ std::unique_ptr<Module> M = parseIR(C, ModStr);
+ auto *Foo = M->getFunction("foo");
+ auto *Goo = M->getFunction("goo");
+
+ auto HashFoo = FunctionHashIgnoringConst::functionHash(*Foo);
+ auto HashGoo = FunctionHashIgnoringConst::functionHash(*Goo);
+
+ // We expect function hashes are matched by ignoring Constant.
+ EXPECT_EQ(HashFoo, HashGoo);
+
+ // IR comparsion is equal when ignoring Constants.
+ GlobalNumberState tGlobalNumbers;
+ FunctionComparatorIgnoringHash FCmp(Foo, Goo, &tGlobalNumbers);
+ EXPECT_EQ(0, FCmp.compareIgnoringConsts());
+
+ // Get the locations and hashes that are different in Constants.
+ using FunctionHash = uint64_t;
+ using IdxHashMapTy = std::map<std::pair<int, int>, FunctionHash>;
+
+ IdxHashMapTy IdxToConstHashFoo, IdxToConstHashGoo;
+ FunctionHashIgnoringConst::functionHash(*Foo, nullptr, &IdxToConstHashFoo);
+ FunctionHashIgnoringConst::functionHash(*Goo, nullptr, &IdxToConstHashGoo);
+ EXPECT_EQ(3, IdxToConstHashFoo.size());
+ EXPECT_EQ(3, IdxToConstHashGoo.size());
+
+ // 0th instruction, 0th operand.
+ // "%1 = load i32, i32* @g1\n" vs. "%1 = load i32, i32* @g2\n"
+ EXPECT_EQ(1, IdxToConstHashFoo.count({0, 0}));
+ EXPECT_EQ(1, IdxToConstHashGoo.count({0, 0}));
+ EXPECT_NE((IdxToConstHashFoo[{0, 0}]), (IdxToConstHashGoo[{0, 0}]));
+
+ // 1st instruction, 1st operand
+ // "store i32 %1, i32* @g1\n" vs. "store i32 %1, i32* @g1\n
+
+ EXPECT_EQ(1, IdxToConstHashFoo.count({1, 1}));
+ EXPECT_EQ(1, IdxToConstHashGoo.count({1, 1}));
+ EXPECT_NE((IdxToConstHashFoo[{1, 1}]), (IdxToConstHashGoo[{1, 1}]));
+
+ // 2nd instruction, 0th operand
+ // "call void @f1()" vs "call void @f2()"
+ EXPECT_EQ(1, IdxToConstHashFoo.count({2, 0}));
+ EXPECT_EQ(1, IdxToConstHashGoo.count({2, 0}));
+ EXPECT_NE((IdxToConstHashFoo[{2, 0}]), (IdxToConstHashGoo[{2, 0}]));
+
+ // Expect the hash of g1 are the same in 0th and 1st instruciton in Foo.
+ EXPECT_EQ((IdxToConstHashFoo[{0, 0}]), (IdxToConstHashFoo[{1, 1}]));
+ // Expect the hash of g2 are the same in 0th and 1st instruciton in Goo.
+ EXPECT_EQ((IdxToConstHashGoo[{0, 0}]), (IdxToConstHashGoo[{1, 1}]));
+ // Expect the hash of g1 is different than that of f1.
+ EXPECT_NE((IdxToConstHashFoo[{0, 0}]), (IdxToConstHashFoo[{2, 0}]));
+}
+
+// This is the case where Constant differs but via `add` operation
+// which isEligibleInstrunctionForConstantSharing does not support.
+TEST(FunctionHashIgnoringConst, HashFail) {
+ LLVMContext C;
+ const char *ModStr = "define i32 @foo(i32 %a) {\n"
+ " %1 = add i32 %a, 1\n"
+ " ret i32 %1\n"
+ "}\n"
+ "define i32 @goo(i32 %a) {\n"
+ " %1 = add i32 %a, 2\n"
+ " ret i32 %1\n"
+ "}\n";
+ std::unique_ptr<Module> M = parseIR(C, ModStr);
+ auto *Foo = M->getFunction("foo");
+ auto *Goo = M->getFunction("goo");
+
+ auto HashFoo = FunctionHashIgnoringConst::functionHash(*Foo);
+ auto HashGoo = FunctionHashIgnoringConst::functionHash(*Goo);
+
+ EXPECT_NE(HashFoo, HashGoo);
+}
More information about the llvm-commits
mailing list