[lld] r315725 - [lld] Move Threads to Common
Bob Haarman via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 13 11:22:55 PDT 2017
Author: inglorion
Date: Fri Oct 13 11:22:55 2017
New Revision: 315725
URL: http://llvm.org/viewvc/llvm-project?rev=315725&view=rev
Log:
[lld] Move Threads to Common
Summary:
This will allow using the functionality from other linkers. It is also
a prerequisite for sharing the error logging code.
Reviewers: ruiu
Reviewed By: ruiu
Subscribers: emaste, mgorny, llvm-commits
Differential Revision: https://reviews.llvm.org/D38822
Added:
lld/trunk/Common/Threads.cpp
lld/trunk/include/lld/Common/Threads.h
Removed:
lld/trunk/ELF/Threads.cpp
lld/trunk/ELF/Threads.h
Modified:
lld/trunk/Common/CMakeLists.txt
lld/trunk/ELF/CMakeLists.txt
lld/trunk/ELF/Config.h
lld/trunk/ELF/Driver.cpp
lld/trunk/ELF/Error.cpp
lld/trunk/ELF/Filesystem.cpp
lld/trunk/ELF/ICF.cpp
lld/trunk/ELF/LinkerScript.cpp
lld/trunk/ELF/MapFile.cpp
lld/trunk/ELF/OutputSections.cpp
lld/trunk/ELF/SyntheticSections.cpp
lld/trunk/ELF/Writer.cpp
Modified: lld/trunk/Common/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/Common/CMakeLists.txt?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/Common/CMakeLists.txt (original)
+++ lld/trunk/Common/CMakeLists.txt Fri Oct 13 11:22:55 2017
@@ -5,6 +5,7 @@ endif()
add_lld_library(lldCommon
Reproduce.cpp
TargetOptionsCommandFlags.cpp
+ Threads.cpp
Version.cpp
ADDITIONAL_HEADER_DIRS
Added: lld/trunk/Common/Threads.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/Common/Threads.cpp?rev=315725&view=auto
==============================================================================
--- lld/trunk/Common/Threads.cpp (added)
+++ lld/trunk/Common/Threads.cpp Fri Oct 13 11:22:55 2017
@@ -0,0 +1,31 @@
+//===- Threads.cpp --------------------------------------------------------===//
+//
+// The LLVM Linker
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "lld/Common/Threads.h"
+#include <thread>
+
+static std::vector<std::thread> Threads;
+
+bool lld::ThreadsEnabled = true;
+
+// Runs a given function in a new thread.
+void lld::runBackground(std::function<void()> Fn) {
+ Threads.emplace_back(Fn);
+}
+
+// Wait for all threads spawned for runBackground() to finish.
+//
+// You need to call this function from the main thread before exiting
+// because it is not defined what will happen to non-main threads when
+// the main thread exits.
+void lld::waitForBackgroundThreads() {
+ for (std::thread &T : Threads)
+ if (T.joinable())
+ T.join();
+}
Modified: lld/trunk/ELF/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/CMakeLists.txt?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/CMakeLists.txt (original)
+++ lld/trunk/ELF/CMakeLists.txt Fri Oct 13 11:22:55 2017
@@ -40,7 +40,6 @@ add_lld_library(lldELF
Symbols.cpp
SyntheticSections.cpp
Target.cpp
- Threads.cpp
Thunks.cpp
Writer.cpp
Modified: lld/trunk/ELF/Config.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Config.h?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/Config.h (original)
+++ lld/trunk/ELF/Config.h Fri Oct 13 11:22:55 2017
@@ -139,7 +139,6 @@ struct Configuration {
bool Static = false;
bool SysvHash = false;
bool Target1Rel;
- bool Threads;
bool Trace;
bool Verbose;
bool WarnCommon;
Modified: lld/trunk/ELF/Driver.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Driver.cpp?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/Driver.cpp (original)
+++ lld/trunk/ELF/Driver.cpp Fri Oct 13 11:22:55 2017
@@ -38,9 +38,9 @@
#include "SymbolTable.h"
#include "SyntheticSections.h"
#include "Target.h"
-#include "Threads.h"
#include "Writer.h"
#include "lld/Common/Driver.h"
+#include "lld/Common/Threads.h"
#include "lld/Common/Version.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
@@ -684,7 +684,7 @@ void LinkerDriver::readConfigs(opt::Inpu
parseCachePruningPolicy(Args.getLastArgValue(OPT_thinlto_cache_policy)),
"--thinlto-cache-policy: invalid cache policy");
Config->ThinLTOJobs = getInteger(Args, OPT_thinlto_jobs, -1u);
- Config->Threads = getArg(Args, OPT_threads, OPT_no_threads, true);
+ ThreadsEnabled = getArg(Args, OPT_threads, OPT_no_threads, true);
Config->Trace = Args.hasArg(OPT_trace);
Config->Undefined = getArgs(Args, OPT_undefined);
Config->UnresolvedSymbols = getUnresolvedSymbolPolicy(Args);
Modified: lld/trunk/ELF/Error.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Error.cpp?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/Error.cpp (original)
+++ lld/trunk/ELF/Error.cpp Fri Oct 13 11:22:55 2017
@@ -9,7 +9,8 @@
#include "Error.h"
#include "Config.h"
-#include "Threads.h"
+
+#include "lld/Common/Threads.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Support/Error.h"
Modified: lld/trunk/ELF/Filesystem.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Filesystem.cpp?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/Filesystem.cpp (original)
+++ lld/trunk/ELF/Filesystem.cpp Fri Oct 13 11:22:55 2017
@@ -13,7 +13,7 @@
#include "Filesystem.h"
#include "Config.h"
-#include "Threads.h"
+#include "lld/Common/Threads.h"
#include "llvm/Support/FileOutputBuffer.h"
#include "llvm/Support/FileSystem.h"
@@ -38,7 +38,7 @@ using namespace lld::elf;
// This function spawns a background thread to call unlink.
// The calling thread returns almost immediately.
void elf::unlinkAsync(StringRef Path) {
- if (!Config->Threads || !sys::fs::exists(Config->OutputFile) ||
+ if (!ThreadsEnabled || !sys::fs::exists(Config->OutputFile) ||
!sys::fs::is_regular_file(Config->OutputFile))
return;
Modified: lld/trunk/ELF/ICF.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/ICF.cpp?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/ICF.cpp (original)
+++ lld/trunk/ELF/ICF.cpp Fri Oct 13 11:22:55 2017
@@ -76,7 +76,7 @@
#include "ICF.h"
#include "Config.h"
#include "SymbolTable.h"
-#include "Threads.h"
+#include "lld/Common/Threads.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/BinaryFormat/ELF.h"
#include "llvm/Object/ELF.h"
@@ -366,7 +366,7 @@ template <class ELFT>
void ICF<ELFT>::forEachClass(std::function<void(size_t, size_t)> Fn) {
// If threading is disabled or the number of sections are
// too small to use threading, call Fn sequentially.
- if (!Config->Threads || Sections.size() < 1024) {
+ if (!ThreadsEnabled || Sections.size() < 1024) {
forEachClassRange(0, Sections.size(), Fn);
++Cnt;
return;
Modified: lld/trunk/ELF/LinkerScript.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/LinkerScript.cpp?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/LinkerScript.cpp (original)
+++ lld/trunk/ELF/LinkerScript.cpp Fri Oct 13 11:22:55 2017
@@ -21,8 +21,8 @@
#include "Symbols.h"
#include "SyntheticSections.h"
#include "Target.h"
-#include "Threads.h"
#include "Writer.h"
+#include "lld/Common/Threads.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/BinaryFormat/ELF.h"
Modified: lld/trunk/ELF/MapFile.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/MapFile.cpp?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/MapFile.cpp (original)
+++ lld/trunk/ELF/MapFile.cpp Fri Oct 13 11:22:55 2017
@@ -26,7 +26,8 @@
#include "Strings.h"
#include "SymbolTable.h"
#include "SyntheticSections.h"
-#include "Threads.h"
+
+#include "lld/Common/Threads.h"
#include "llvm/Support/raw_ostream.h"
Modified: lld/trunk/ELF/OutputSections.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/OutputSections.cpp?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/OutputSections.cpp (original)
+++ lld/trunk/ELF/OutputSections.cpp Fri Oct 13 11:22:55 2017
@@ -15,7 +15,7 @@
#include "SymbolTable.h"
#include "SyntheticSections.h"
#include "Target.h"
-#include "Threads.h"
+#include "lld/Common/Threads.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/Support/Compression.h"
#include "llvm/Support/MD5.h"
Modified: lld/trunk/ELF/SyntheticSections.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/SyntheticSections.cpp?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/SyntheticSections.cpp (original)
+++ lld/trunk/ELF/SyntheticSections.cpp Fri Oct 13 11:22:55 2017
@@ -24,8 +24,8 @@
#include "Strings.h"
#include "SymbolTable.h"
#include "Target.h"
-#include "Threads.h"
#include "Writer.h"
+#include "lld/Common/Threads.h"
#include "lld/Common/Version.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/DebugInfo/DWARF/DWARFDebugPubTable.h"
@@ -2258,7 +2258,7 @@ void MergeNoTailSection::finalizeContent
// Concurrency level. Must be a power of 2 to avoid expensive modulo
// operations in the following tight loop.
size_t Concurrency = 1;
- if (Config->Threads)
+ if (ThreadsEnabled)
Concurrency =
std::min<size_t>(PowerOf2Floor(hardware_concurrency()), NumShards);
Removed: lld/trunk/ELF/Threads.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Threads.cpp?rev=315724&view=auto
==============================================================================
--- lld/trunk/ELF/Threads.cpp (original)
+++ lld/trunk/ELF/Threads.cpp (removed)
@@ -1,29 +0,0 @@
-//===- Threads.cpp --------------------------------------------------------===//
-//
-// The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Threads.h"
-#include <thread>
-
-static std::vector<std::thread> Threads;
-
-// Runs a given function in a new thread.
-void lld::elf::runBackground(std::function<void()> Fn) {
- Threads.emplace_back(Fn);
-}
-
-// Wait for all threads spawned for runBackground() to finish.
-//
-// You need to call this function from the main thread before exiting
-// because it is not defined what will happen to non-main threads when
-// the main thread exits.
-void lld::elf::waitForBackgroundThreads() {
- for (std::thread &T : Threads)
- if (T.joinable())
- T.join();
-}
Removed: lld/trunk/ELF/Threads.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Threads.h?rev=315724&view=auto
==============================================================================
--- lld/trunk/ELF/Threads.h (original)
+++ lld/trunk/ELF/Threads.h (removed)
@@ -1,91 +0,0 @@
-//===- Threads.h ------------------------------------------------*- C++ -*-===//
-//
-// The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// LLD supports threads to distribute workloads to multiple cores. Using
-// multicore is most effective when more than one core are idle. At the
-// last step of a build, it is often the case that a linker is the only
-// active process on a computer. So, we are naturally interested in using
-// threads wisely to reduce latency to deliver results to users.
-//
-// That said, we don't want to do "too clever" things using threads.
-// Complex multi-threaded algorithms are sometimes extremely hard to
-// reason about and can easily mess up the entire design.
-//
-// Fortunately, when a linker links large programs (when the link time is
-// most critical), it spends most of the time to work on massive number of
-// small pieces of data of the same kind, and there are opportunities for
-// large parallelism there. Here are examples:
-//
-// - We have hundreds of thousands of input sections that need to be
-// copied to a result file at the last step of link. Once we fix a file
-// layout, each section can be copied to its destination and its
-// relocations can be applied independently.
-//
-// - We have tens of millions of small strings when constructing a
-// mergeable string section.
-//
-// For the cases such as the former, we can just use parallelForEach
-// instead of std::for_each (or a plain for loop). Because tasks are
-// completely independent from each other, we can run them in parallel
-// without any coordination between them. That's very easy to understand
-// and reason about.
-//
-// For the cases such as the latter, we can use parallel algorithms to
-// deal with massive data. We have to write code for a tailored algorithm
-// for each problem, but the complexity of multi-threading is isolated in
-// a single pass and doesn't affect the linker's overall design.
-//
-// The above approach seems to be working fairly well. As an example, when
-// linking Chromium (output size 1.6 GB), using 4 cores reduces latency to
-// 75% compared to single core (from 12.66 seconds to 9.55 seconds) on my
-// Ivy Bridge Xeon 2.8 GHz machine. Using 40 cores reduces it to 63% (from
-// 12.66 seconds to 7.95 seconds). Because of the Amdahl's law, the
-// speedup is not linear, but as you add more cores, it gets faster.
-//
-// On a final note, if you are trying to optimize, keep the axiom "don't
-// guess, measure!" in mind. Some important passes of the linker are not
-// that slow. For example, resolving all symbols is not a very heavy pass,
-// although it would be very hard to parallelize it. You want to first
-// identify a slow pass and then optimize it.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLD_ELF_THREADS_H
-#define LLD_ELF_THREADS_H
-
-#include "Config.h"
-
-#include "llvm/Support/Parallel.h"
-#include <functional>
-
-namespace lld {
-namespace elf {
-
-template <typename R, class FuncTy> void parallelForEach(R &&Range, FuncTy Fn) {
- if (Config->Threads)
- for_each(llvm::parallel::par, std::begin(Range), std::end(Range), Fn);
- else
- for_each(llvm::parallel::seq, std::begin(Range), std::end(Range), Fn);
-}
-
-inline void parallelForEachN(size_t Begin, size_t End,
- std::function<void(size_t)> Fn) {
- if (Config->Threads)
- for_each_n(llvm::parallel::par, Begin, End, Fn);
- else
- for_each_n(llvm::parallel::seq, Begin, End, Fn);
-}
-
-void runBackground(std::function<void()> Fn);
-void waitForBackgroundThreads();
-
-} // namespace elf
-} // namespace lld
-
-#endif
Modified: lld/trunk/ELF/Writer.cpp
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/ELF/Writer.cpp?rev=315725&r1=315724&r2=315725&view=diff
==============================================================================
--- lld/trunk/ELF/Writer.cpp (original)
+++ lld/trunk/ELF/Writer.cpp Fri Oct 13 11:22:55 2017
@@ -19,7 +19,7 @@
#include "SymbolTable.h"
#include "SyntheticSections.h"
#include "Target.h"
-#include "Threads.h"
+#include "lld/Common/Threads.h"
#include "llvm/ADT/StringMap.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Support/FileOutputBuffer.h"
Added: lld/trunk/include/lld/Common/Threads.h
URL: http://llvm.org/viewvc/llvm-project/lld/trunk/include/lld/Common/Threads.h?rev=315725&view=auto
==============================================================================
--- lld/trunk/include/lld/Common/Threads.h (added)
+++ lld/trunk/include/lld/Common/Threads.h Fri Oct 13 11:22:55 2017
@@ -0,0 +1,89 @@
+//===- Threads.h ------------------------------------------------*- C++ -*-===//
+//
+// The LLVM Linker
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// LLD supports threads to distribute workloads to multiple cores. Using
+// multicore is most effective when more than one core are idle. At the
+// last step of a build, it is often the case that a linker is the only
+// active process on a computer. So, we are naturally interested in using
+// threads wisely to reduce latency to deliver results to users.
+//
+// That said, we don't want to do "too clever" things using threads.
+// Complex multi-threaded algorithms are sometimes extremely hard to
+// reason about and can easily mess up the entire design.
+//
+// Fortunately, when a linker links large programs (when the link time is
+// most critical), it spends most of the time to work on massive number of
+// small pieces of data of the same kind, and there are opportunities for
+// large parallelism there. Here are examples:
+//
+// - We have hundreds of thousands of input sections that need to be
+// copied to a result file at the last step of link. Once we fix a file
+// layout, each section can be copied to its destination and its
+// relocations can be applied independently.
+//
+// - We have tens of millions of small strings when constructing a
+// mergeable string section.
+//
+// For the cases such as the former, we can just use parallelForEach
+// instead of std::for_each (or a plain for loop). Because tasks are
+// completely independent from each other, we can run them in parallel
+// without any coordination between them. That's very easy to understand
+// and reason about.
+//
+// For the cases such as the latter, we can use parallel algorithms to
+// deal with massive data. We have to write code for a tailored algorithm
+// for each problem, but the complexity of multi-threading is isolated in
+// a single pass and doesn't affect the linker's overall design.
+//
+// The above approach seems to be working fairly well. As an example, when
+// linking Chromium (output size 1.6 GB), using 4 cores reduces latency to
+// 75% compared to single core (from 12.66 seconds to 9.55 seconds) on my
+// Ivy Bridge Xeon 2.8 GHz machine. Using 40 cores reduces it to 63% (from
+// 12.66 seconds to 7.95 seconds). Because of the Amdahl's law, the
+// speedup is not linear, but as you add more cores, it gets faster.
+//
+// On a final note, if you are trying to optimize, keep the axiom "don't
+// guess, measure!" in mind. Some important passes of the linker are not
+// that slow. For example, resolving all symbols is not a very heavy pass,
+// although it would be very hard to parallelize it. You want to first
+// identify a slow pass and then optimize it.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_COMMON_THREADS_H
+#define LLD_COMMON_THREADS_H
+
+#include "llvm/Support/Parallel.h"
+#include <functional>
+
+namespace lld {
+
+extern bool ThreadsEnabled;
+
+template <typename R, class FuncTy> void parallelForEach(R &&Range, FuncTy Fn) {
+ if (ThreadsEnabled)
+ for_each(llvm::parallel::par, std::begin(Range), std::end(Range), Fn);
+ else
+ for_each(llvm::parallel::seq, std::begin(Range), std::end(Range), Fn);
+}
+
+inline void parallelForEachN(size_t Begin, size_t End,
+ std::function<void(size_t)> Fn) {
+ if (ThreadsEnabled)
+ for_each_n(llvm::parallel::par, Begin, End, Fn);
+ else
+ for_each_n(llvm::parallel::seq, Begin, End, Fn);
+}
+
+void runBackground(std::function<void()> Fn);
+void waitForBackgroundThreads();
+
+} // namespace lld
+
+#endif
More information about the llvm-commits
mailing list