[clang] [llvm] [CMake][PGO] Add option for using an external project to generate profile data (PR #78879)
Tom Stellard via cfe-commits
cfe-commits at lists.llvm.org
Tue Jan 23 22:43:57 PST 2024
https://github.com/tstellar updated https://github.com/llvm/llvm-project/pull/78879
>From 0719f49ecc6dd69ae4698c3e84dbf175a1bf2ed3 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sun, 21 Jan 2024 00:25:47 +0000
Subject: [PATCH 1/4] [CMake][PGO] Add libunwind to list of stage1 runtimes
This fixes the build since 8f90e6937a1fac80873bb2dab5f382c82ba1ba4e
which made libcxxabi use llvm's libunwind by default.
---
clang/cmake/caches/PGO.cmake | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/clang/cmake/caches/PGO.cmake b/clang/cmake/caches/PGO.cmake
index e1d0585e453f825..15bc755d110d19a 100644
--- a/clang/cmake/caches/PGO.cmake
+++ b/clang/cmake/caches/PGO.cmake
@@ -2,7 +2,7 @@ set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "")
set(CLANG_ENABLE_BOOTSTRAP ON CACHE BOOL "")
set(LLVM_ENABLE_PROJECTS "clang;lld" CACHE STRING "")
-set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi" CACHE STRING "")
+set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi;libunwind" CACHE STRING "")
set(LLVM_TARGETS_TO_BUILD Native CACHE STRING "")
set(BOOTSTRAP_LLVM_BUILD_INSTRUMENTED ON CACHE BOOL "")
>From 5c602233ef4e54e850f6c8a17c25968bc706b898 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Sat, 6 Jan 2024 07:46:01 +0000
Subject: [PATCH 2/4] [CMake][PGO] Add option for using an external project to
generate profile data.
The new CLANG_PGO_TRAINING_DATA_SOURCE_DIR allows users to specify a CMake
project to use for generating the profile data.
For example, to use the llvm-test-suite to generate profile data you
would do:
$ cmake -G Ninja -B build -S llvm -C <path to source>/clang/cmake/caches/PGO.cmake \
-DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=<path to llvm-test-suite> \
-DBOOTSTRAP_CLANG_PERF_TRAINING_DEPS=runtimes
---
clang/utils/perf-training/CMakeLists.txt | 13 +++++++++++--
clang/utils/perf-training/perf-helper.py | 16 +++++++++-------
llvm/docs/AdvancedBuilds.rst | 23 +++++++++++++++++++++++
3 files changed, 43 insertions(+), 9 deletions(-)
diff --git a/clang/utils/perf-training/CMakeLists.txt b/clang/utils/perf-training/CMakeLists.txt
index c6d51863fb1b5c2..f9d673b2e92e775 100644
--- a/clang/utils/perf-training/CMakeLists.txt
+++ b/clang/utils/perf-training/CMakeLists.txt
@@ -1,6 +1,10 @@
+include(LLVMExternalProjectUtils)
+
set(CLANG_PGO_TRAINING_DATA "${CMAKE_CURRENT_SOURCE_DIR}" CACHE PATH
"The path to a lit testsuite containing samples for PGO and order file generation"
)
+set(CLANG_PGO_TRAINING_DATA_SOURCE_DIR OFF CACHE STRING "Path to source directory containing cmake project with source files to use for generating pgo data")
+set(CLANG_PERF_TRAINING_DEPS "" CACHE STRING "Extra dependencies needed to build the PGO training data.")
if(LLVM_BUILD_INSTRUMENTED)
configure_lit_site_cfg(
@@ -15,7 +19,7 @@ if(LLVM_BUILD_INSTRUMENTED)
)
add_custom_target(clear-profraw
- COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py clean ${CMAKE_CURRENT_BINARY_DIR} profraw
+ COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py clean ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}/profiles/ profraw
COMMENT "Clearing old profraw data")
if(NOT LLVM_PROFDATA)
@@ -26,9 +30,14 @@ if(LLVM_BUILD_INSTRUMENTED)
message(STATUS "To enable merging PGO data LLVM_PROFDATA has to point to llvm-profdata")
else()
add_custom_target(generate-profdata
- COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge ${LLVM_PROFDATA} ${CMAKE_CURRENT_BINARY_DIR}/clang.profdata ${CMAKE_CURRENT_BINARY_DIR}
+ COMMAND "${Python3_EXECUTABLE}" ${CMAKE_CURRENT_SOURCE_DIR}/perf-helper.py merge ${LLVM_PROFDATA} ${CMAKE_CURRENT_BINARY_DIR}/clang.profdata ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_BINARY_DIR}/profiles/
COMMENT "Merging profdata"
DEPENDS generate-profraw)
+ if (CLANG_PGO_TRAINING_DATA_SOURCE_DIR)
+ llvm_ExternalProject_Add(generate-profraw-external ${CLANG_PGO_TRAINING_DATA_SOURCE_DIR}
+ USE_TOOLCHAIN EXLUDE_FROM_ALL NO_INSTALL DEPENDS generate-profraw)
+ add_dependencies(generate-profdata generate-profraw-external)
+ endif()
endif()
endif()
diff --git a/clang/utils/perf-training/perf-helper.py b/clang/utils/perf-training/perf-helper.py
index 99d6a3333b6ef08..844aa274f049aaa 100644
--- a/clang/utils/perf-training/perf-helper.py
+++ b/clang/utils/perf-training/perf-helper.py
@@ -30,26 +30,28 @@ def findFilesWithExtension(path, extension):
def clean(args):
- if len(args) != 2:
+ if len(args) < 2:
print(
- "Usage: %s clean <path> <extension>\n" % __file__
+ "Usage: %s clean <paths> <extension>\n" % __file__
+ "\tRemoves all files with extension from <path>."
)
return 1
- for filename in findFilesWithExtension(args[0], args[1]):
- os.remove(filename)
+ for path in args[1:-1]:
+ for filename in findFilesWithExtension(path, args[-1]):
+ os.remove(filename)
return 0
def merge(args):
- if len(args) != 3:
+ if len(args) < 3:
print(
- "Usage: %s merge <llvm-profdata> <output> <path>\n" % __file__
+ "Usage: %s merge <llvm-profdata> <output> <paths>\n" % __file__
+ "\tMerges all profraw files from path into output."
)
return 1
cmd = [args[0], "merge", "-o", args[1]]
- cmd.extend(findFilesWithExtension(args[2], "profraw"))
+ for i in range(2, len(args)):
+ cmd.extend(findFilesWithExtension(args[i], "profraw"))
subprocess.check_call(cmd)
return 0
diff --git a/llvm/docs/AdvancedBuilds.rst b/llvm/docs/AdvancedBuilds.rst
index 960b19fa5317f3b..3ff630a1425691d 100644
--- a/llvm/docs/AdvancedBuilds.rst
+++ b/llvm/docs/AdvancedBuilds.rst
@@ -145,6 +145,29 @@ that also enables ThinTLO, use the following command:
-DPGO_INSTRUMENT_LTO=Thin \
<path to source>/llvm
+By default, clang will generate profile data by compiling a simple
+hello world program. You can also tell clang use an external
+project for generating profile data that may be a better fit for your
+use case. The project you specify must either be a lit test suite
+(use the CLANG_PGO_TRAINING_DATA option) or a CMake project (use the
+CLANG_PERF_TRAINING_DATA_SOURCE_DIR option).
+
+For example, If you wanted to use the
+`LLVM Test Suite <https://github.com/llvm/llvm-test-suite/>`_ to generate
+profile data you would use the following command:
+
+.. code-block:: console
+
+ $ cmake -G Ninja -C <path to source>/clang/cmake/caches/PGO.cmake \
+ -DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=<path to llvm-test-suite> \
+ -DBOOTSTRAP_CLANG_PERF_TRAINING_DEPS=runtimes
+
+The BOOTSTRAP\_ prefixes tells CMake to pass the variables on to the instrumented
+stage two build. And the CLANG_PERF_TRAINING_DEPS option let's you specify
+additional build targets to build before building the external project. The
+LLVM Test Suite requires compiler-rt to build, so we need to add the
+`runtimes` target as a dependency.
+
After configuration, building the stage2-instrumented-generate-profdata target
will automatically build the stage1 compiler, build the instrumented compiler
with the stage1 compiler, and then run the instrumented compiler against the
>From 4f9fa290a6fd7bf4319e55da5d6f3ce3de7f31a4 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Tue, 23 Jan 2024 11:29:19 -0800
Subject: [PATCH 3/4] Update clang/utils/perf-training/perf-helper.py
Co-authored-by: Petr Hosek <phosek at google.com>
---
clang/utils/perf-training/perf-helper.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/utils/perf-training/perf-helper.py b/clang/utils/perf-training/perf-helper.py
index 844aa274f049aaa..3e92cd38a714515 100644
--- a/clang/utils/perf-training/perf-helper.py
+++ b/clang/utils/perf-training/perf-helper.py
@@ -50,8 +50,8 @@ def merge(args):
)
return 1
cmd = [args[0], "merge", "-o", args[1]]
- for i in range(2, len(args)):
- cmd.extend(findFilesWithExtension(args[i], "profraw"))
+ for path in args[2:]:
+ cmd.extend(findFilesWithExtension(path, "profraw"))
subprocess.check_call(cmd)
return 0
>From 552d25160abd4b1311c7cfa4299d8ee199d12113 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar at redhat.com>
Date: Wed, 24 Jan 2024 04:31:04 +0000
Subject: [PATCH 4/4] Rename CLANG_PERF_TRAINING_DEPS to
CLANG_PGO_TRAINING_DEPS
---
clang/utils/perf-training/CMakeLists.txt | 4 ++--
llvm/docs/AdvancedBuilds.rst | 8 ++++----
2 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/clang/utils/perf-training/CMakeLists.txt b/clang/utils/perf-training/CMakeLists.txt
index f9d673b2e92e775..93744f460602365 100644
--- a/clang/utils/perf-training/CMakeLists.txt
+++ b/clang/utils/perf-training/CMakeLists.txt
@@ -4,7 +4,7 @@ set(CLANG_PGO_TRAINING_DATA "${CMAKE_CURRENT_SOURCE_DIR}" CACHE PATH
"The path to a lit testsuite containing samples for PGO and order file generation"
)
set(CLANG_PGO_TRAINING_DATA_SOURCE_DIR OFF CACHE STRING "Path to source directory containing cmake project with source files to use for generating pgo data")
-set(CLANG_PERF_TRAINING_DEPS "" CACHE STRING "Extra dependencies needed to build the PGO training data.")
+set(CLANG_PGO_TRAINING_DEPS "" CACHE STRING "Extra dependencies needed to build the PGO training data.")
if(LLVM_BUILD_INSTRUMENTED)
configure_lit_site_cfg(
@@ -15,7 +15,7 @@ if(LLVM_BUILD_INSTRUMENTED)
add_lit_testsuite(generate-profraw "Generating clang PGO data"
${CMAKE_CURRENT_BINARY_DIR}/pgo-data/
EXCLUDE_FROM_CHECK_ALL
- DEPENDS clang clear-profraw ${CLANG_PERF_TRAINING_DEPS}
+ DEPENDS clang clear-profraw ${CLANG_PGO_TRAINING_DEPS}
)
add_custom_target(clear-profraw
diff --git a/llvm/docs/AdvancedBuilds.rst b/llvm/docs/AdvancedBuilds.rst
index 3ff630a1425691d..ee178dd3772c4b4 100644
--- a/llvm/docs/AdvancedBuilds.rst
+++ b/llvm/docs/AdvancedBuilds.rst
@@ -160,10 +160,10 @@ profile data you would use the following command:
$ cmake -G Ninja -C <path to source>/clang/cmake/caches/PGO.cmake \
-DBOOTSTRAP_CLANG_PGO_TRAINING_DATA_SOURCE_DIR=<path to llvm-test-suite> \
- -DBOOTSTRAP_CLANG_PERF_TRAINING_DEPS=runtimes
+ -DBOOTSTRAP_CLANG_PGO_TRAINING_DEPS=runtimes
The BOOTSTRAP\_ prefixes tells CMake to pass the variables on to the instrumented
-stage two build. And the CLANG_PERF_TRAINING_DEPS option let's you specify
+stage two build. And the CLANG_PGO_TRAINING_DEPS option let's you specify
additional build targets to build before building the external project. The
LLVM Test Suite requires compiler-rt to build, so we need to add the
`runtimes` target as a dependency.
@@ -195,12 +195,12 @@ You can feed that file into the LLVM_PROFDATA_FILE option when you build your
optimized compiler.
It may be necessary to build additional targets before running perf training, such as
-builtins and runtime libraries. You can use the :code:`CLANG_PERF_TRAINING_DEPS` CMake
+builtins and runtime libraries. You can use the :code:`CLANG_PGO_TRAINING_DEPS` CMake
variable for that purpose:
.. code-block:: cmake
- set(CLANG_PERF_TRAINING_DEPS builtins runtimes CACHE STRING "")
+ set(CLANG_PGO_TRAINING_DEPS builtins runtimes CACHE STRING "")
The PGO cache has a slightly different stage naming scheme than other
multi-stage builds. It generates three stages: stage1, stage2-instrumented, and
More information about the cfe-commits
mailing list