[clang] [llvm] [CMake][Release] Add option for enabling LTO to cache file (PR #77035)
Tom Stellard via cfe-commits
cfe-commits at lists.llvm.org
Mon Jan 8 09:32:32 PST 2024
https://github.com/tstellar updated https://github.com/llvm/llvm-project/pull/77035
>From db643899042aea45da93c1738b523f139f307295 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Fri, 5 Jan 2024 00:58:58 +0000
Subject: [PATCH 1/3] [CMake][Release] Add option for enabling LTO to cache
file
This option is LLVM_RELEASE_ENABLE_LTO and it's turned on by default.
---
clang/cmake/caches/Release.cmake | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/clang/cmake/caches/Release.cmake b/clang/cmake/caches/Release.cmake
index 3ea65ce26296c8..a7b9a8d0e29f88 100644
--- a/clang/cmake/caches/Release.cmake
+++ b/clang/cmake/caches/Release.cmake
@@ -2,6 +2,9 @@
# BOOTSTRAP_* options configure the second build.
# BOOTSTRAP_BOOTSTRAP_* options configure the third build.
+# General Options
+set(LLVM_RELEASE_ENABLE_LTO THIN CACHE STRING "")
+
set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "")
# Stage 1 Bootstrap Setup
@@ -33,9 +36,17 @@ set(BOOTSTRAP_CLANG_BOOTSTRAP_TARGETS
check-clang CACHE STRING "")
# Stage 2 Options
-set(BOOTSTRAP_LLVM_ENABLE_PROJECTS "clang" CACHE STRING "")
+set(STAGE2_PROJECTS "clang")
+if (LLVM_RELEASE_ENABLE_LTO)
+ list(APPEND STAGE2_PROJECTS "lld")
+endif()
+set(BOOTSTRAP_LLVM_ENABLE_PROJECTS ${STAGE2_PROJECTS} CACHE STRING "")
set(BOOTSTRAP_LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
# Stage 3 Options
set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_PROJECTS "clang;lld;lldb;clang-tools-extra;bolt;polly;mlir;flang" CACHE STRING "")
+set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LTO ${LLVM_RELEASE_ENABLE_LTO} CACHE STRING "")
+if (LLVM_RELEASE_ENABLE_LTO)
+ set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_LLD ON CACHE BOOL "")
+endif()
>From b41c8bb2dda110674a2c1bdc4daf0ed88d69bfcb Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 6 Jan 2024 06:44:32 +0000
Subject: [PATCH 2/3] XXX: test-release.sh pgo
---
clang/cmake/caches/Release.cmake | 77 +++++++++++++++++++++++-------
llvm/utils/release/test-release.sh | 4 +-
2 files changed, 63 insertions(+), 18 deletions(-)
diff --git a/clang/cmake/caches/Release.cmake b/clang/cmake/caches/Release.cmake
index a7b9a8d0e29f88..1d8d5a7383ac37 100644
--- a/clang/cmake/caches/Release.cmake
+++ b/clang/cmake/caches/Release.cmake
@@ -4,27 +4,57 @@
# General Options
set(LLVM_RELEASE_ENABLE_LTO THIN CACHE STRING "")
+set(LLVM_RELEASE_ENABLE_PGO ON CACHE BOOL "")
set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "")
# Stage 1 Bootstrap Setup
set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "")
-set(CLANG_BOOTSTRAP_TARGETS
- clang
- check-all
- check-llvm
- check-clang
- test-suite
- stage3
- stage3-clang
- stage3-check-all
- stage3-check-llvm
- stage3-check-clang
- stage3-install
- stage3-test-suite CACHE STRING "")
+if (LLVM_RELEASE_ENABLE_PGO)
+ set(CLANG_BOOTSTRAP_TARGETS
+ generate-profdata
+ stage2
+ stage2-clang
+ stage2-distribution
+ stage2-install
+ stage2-install-distribution
+ stage2-install-distribution-toolchain
+ stage2-check-all
+ stage2-check-llvm
+ stage2-check-clang
+ stage2-test-suite CACHE STRING "")
+else()
+ set(CLANG_BOOTSTRAP_TARGETS
+ clang
+ check-all
+ check-llvm
+ check-clang
+ test-suite
+ stage3
+ stage3-clang
+ stage3-check-all
+ stage3-check-llvm
+ stage3-check-clang
+ stage3-install
+ stage3-test-suite CACHE STRING "")
+endif()
+
+if (LLVM_RELEASE_ENABLE_PGO)
+ list(INSERT CLANG_BOOTSTRAP_TARGETS 0 generate-profdata)
+endif()
# Stage 1 Options
-set(LLVM_ENABLE_PROJECTS "clang" CACHE STRING "")
+set(STAGE1_PROJECTS "clang")
+set(STAGE1_RUNTIMES "")
+
+if (LLVM_RELEASE_ENABLE_PGO)
+ list(APPEND STAGE1_PROJECTS "lld")
+ list(APPEND STAGE1_RUNTIMES "compiler-rt")
+endif()
+
+set(LLVM_ENABLE_RUNTIMES ${STAGE1_RUNTIMES} CACHE STRING "")
+set(LLVM_ENABLE_PROJECTS ${STAGE1_PROJECTS} CACHE STRING "")
+
set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
# Stage 2 Bootstrap Setup
@@ -37,11 +67,26 @@ set(BOOTSTRAP_CLANG_BOOTSTRAP_TARGETS
# Stage 2 Options
set(STAGE2_PROJECTS "clang")
-if (LLVM_RELEASE_ENABLE_LTO)
+set(STAGE2_RUNTIMES "")
+
+if (LLVM_RELEASE_ENABLE_LTO OR LLVM_RELEASE_ENABLE_PGO)
list(APPEND STAGE2_PROJECTS "lld")
endif()
+
+if (LLVM_RELEASE_ENABLE_PGO)
+ set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "")
+ list(APPEND STAGE2_RUTNIMES "compiler-rt")
+ set(BOOTSTRAP_LLVM_ENABLE_LTO ${LLVM_RELEASE_ENABLE_LTO})
+ if (LLVM_RELEASE_ENABLE_LTO)
+ set(BOOTSTRAP_LLVM_ENABLE_LLD ON CACHE BOOL "")
+ endif()
+endif()
+
set(BOOTSTRAP_LLVM_ENABLE_PROJECTS ${STAGE2_PROJECTS} CACHE STRING "")
-set(BOOTSTRAP_LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
+set(BOOTSTRAP_LLVM_ENABLE_RUNTIMES ${STAGE2_RUNTIMES} CACHE STRING "")
+if (NOT LLVM_RELEASE_PGO)
+ set(BOOTSTRAP_LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
+endif()
# Stage 3 Options
set(BOOTSTRAP_BOOTSTRAP_LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
diff --git a/llvm/utils/release/test-release.sh b/llvm/utils/release/test-release.sh
index 544d4bfdd799ce..5b1945df47d24a 100755
--- a/llvm/utils/release/test-release.sh
+++ b/llvm/utils/release/test-release.sh
@@ -359,11 +359,11 @@ function build_with_cmake_cache() {
$ExtraConfigureFlags
2>&1 | tee $LogDir/llvm.configure-$Flavor.log
- ${MAKE} $J_ARG $Verbose -C $CMakeBuildDir stage3-check-all \
+ ${MAKE} $J_ARG $Verbose -C $CMakeBuildDir stage2-check-all \
2>&1 | tee $LogDir/llvm.make-$Flavor.log > $redir
DESTDIR="${InstallDir}" \
- ${MAKE} -C $CMakeBuildDir stage3-install \
+ ${MAKE} -C $CMakeBuildDir stage2-install \
2>&1 | tee $LogDir/llvm.install-$Flavor.log > $redir
mkdir -p $BuildDir/Release
>From df8296d418eac2b4db9d955a0aec25c4c1c22a9c Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 6 Jan 2024 07:46:01 +0000
Subject: [PATCH 3/3] [CMake][PGO] Use check-clang target to generate profdata
for PGO builds
When doing a multi-stage PGO build of clang, run the check-clang and
check-llvm targets using the instrumented clang and use that profile
data for building the final stage2 clang. This is what is recommended
by our official documentation: https://llvm.org/docs/HowToBuildWithPGO.html#building-clang-with-pgo
I benchmarked this change by compiling the SemaChecking.cpp file from
clang. Using check-clang/check-llvm to generate the profile data gives a 25% speedup
in the PGO+LTO stage2 clang when compared to the stage1 clang (no-LTO).
Prior to this change, I was only seeing ~5% speedup when comparing the
stage2 and stage1 builds.
---
clang/utils/perf-training/CMakeLists.txt | 6 +++---
clang/utils/perf-training/perf-helper.py | 16 +++++++++-------
2 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/clang/utils/perf-training/CMakeLists.txt b/clang/utils/perf-training/CMakeLists.txt
index c6d51863fb1b5c..95ff8115aa538b 100644
--- a/clang/utils/perf-training/CMakeLists.txt
+++ b/clang/utils/perf-training/CMakeLists.txt
@@ -15,7 +15,7 @@ if(LLVM_BUILD_INSTRUMENTED)
)
add_custom_target(clear-profraw
- COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py clean ${CMAKE_CURRENT_BINARY_DIR} profraw
+ COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py clean ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}/profiles/ profraw
COMMENT "Clearing old profraw data")
if(NOT LLVM_PROFDATA)
@@ -26,9 +26,9 @@ if(LLVM_BUILD_INSTRUMENTED)
message(STATUS "To enable merging PGO data LLVM_PROFDATA has to point to llvm-profdata")
else()
add_custom_target(generate-profdata
- COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge ${LLVM_PROFDATA} ${CMAKE_CURRENT_BINARY_DIR}/clang.profdata ${CMAKE_CURRENT_BINARY_DIR}
+ COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge ${LLVM_PROFDATA} ${CMAKE_CURRENT_BINARY_DIR}/clang.profdata ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}/profiles/
COMMENT "Merging profdata"
- DEPENDS generate-profraw)
+ DEPENDS generate-profraw check-clang check-llvm)
endif()
endif()
diff --git a/clang/utils/perf-training/perf-helper.py b/clang/utils/perf-training/perf-helper.py
index 99d6a3333b6ef0..bd8f74c9c2e129 100644
--- a/clang/utils/perf-training/perf-helper.py
+++ b/clang/utils/perf-training/perf-helper.py
@@ -30,26 +30,28 @@ def findFilesWithExtension(path, extension):
def clean(args):
- if len(args) != 2:
+ if len(args) < 2:
print(
- "Usage: %s clean <path> <extension>\n" % __file__
+ "Usage: %s clean <paths> <extension>\n" % __file__
+ "\tRemoves all files with extension from <path>."
)
return 1
- for filename in findFilesWithExtension(args[0], args[1]):
- os.remove(filename)
+ for path in args[1:-1]:
+ for filename in findFilesWithExtension(path, args[-1]):
+ os.remove(filename)
return 0
def merge(args):
- if len(args) != 3:
+ if len(args) < 3:
print(
- "Usage: %s merge <llvm-profdata> <output> <path>\n" % __file__
+ "Usage: %s merge <llvm-profdata> <output> <paths>\n" % __file__
+ "\tMerges all profraw files from path into output."
)
return 1
cmd = [args[0], "merge", "-o", args[1]]
- cmd.extend(findFilesWithExtension(args[2], "profraw"))
+ for i in range(2, len(args)):
+ cmd.extend(findFilesWithExtension(args[i], "profraw"))
subprocess.check_call(cmd)
return 0
More information about the cfe-commits
mailing list