[Openmp-commits] [openmp] 5b67bce - [OpenMP] [OMPT] [2/8] Implemented a connector for communication of OMPT callbacks between libraries.
Dhruva Chakrabarti via Openmp-commits
openmp-commits at lists.llvm.org
Tue Nov 15 14:22:18 PST 2022
Author: Dhruva Chakrabarti
Date: 2022-11-15T14:21:55-08:00
New Revision: 5b67bce787e4eeee1222c78bee7c0e9bc5563a3a
URL: https://github.com/llvm/llvm-project/commit/5b67bce787e4eeee1222c78bee7c0e9bc5563a3a
DIFF: https://github.com/llvm/llvm-project/commit/5b67bce787e4eeee1222c78bee7c0e9bc5563a3a.diff
LOG: [OpenMP] [OMPT] [2/8] Implemented a connector for communication of OMPT callbacks between libraries.
This is part of a set of patches implementing OMPT target callback support and has been split out of the originally submitted https://reviews.llvm.org/D113728. The overall design can be found in https://rice.app.box.com/s/pf3gix2hs4d4o1aatwir1set05xmjljc
The purpose of this patch is to provide a way to register tool-provided callbacks into libomp when libomptarget is loaded.
Introduced a cmake variable LIBOMPTARGET_OMPT_SUPPORT that can be used to control OMPT target support. It follows host OMPT support, controlled by LIBOMP_HAVE_OMPT_SUPPORT.
Added a connector that can be used to communicate between OMPT implementations in libomp and libomptarget or libomptarget and a plugin.
Added a global constructor in libomptarget that uses the connector to force registration of tool-provided callbacks in libomp. A pair of init and fini functions are provided to libomp as part of the connect process which will be used to register the tool-provided callbacks in libomptarget.
Patch from John Mellor-Crummey <johnmc at rice.edu>
(With contributions from Dhruva Chakrabarti <Dhruva.Chakrabarti at amd.com>)
Reviewed By: dreachem, jhuber6
Differential Revision: https://reviews.llvm.org/D123572
Added:
openmp/libomptarget/include/ompt_connector.h
openmp/libomptarget/src/ompt_callback.cpp
Modified:
openmp/libomptarget/CMakeLists.txt
openmp/libomptarget/src/CMakeLists.txt
openmp/runtime/CMakeLists.txt
openmp/runtime/cmake/config-ix.cmake
openmp/runtime/src/CMakeLists.txt
openmp/runtime/src/exports_so.txt
openmp/runtime/src/ompt-general.cpp
openmp/runtime/src/ompt-specific.cpp
openmp/runtime/src/ompt-specific.h
Removed:
################################################################################
diff --git a/openmp/libomptarget/CMakeLists.txt b/openmp/libomptarget/CMakeLists.txt
index a99e6611c58ca..21f211ad9d1ad 100644
--- a/openmp/libomptarget/CMakeLists.txt
+++ b/openmp/libomptarget/CMakeLists.txt
@@ -69,7 +69,28 @@ if(LIBOMPTARGET_ENABLE_DEBUG)
add_definitions(-DOMPTARGET_DEBUG)
endif()
+# OMPT support for libomptarget
+# Follow host OMPT support and check if host support has been requested.
+# LIBOMP_HAVE_OMPT_SUPPORT indicates whether host OMPT support has been implemented.
+# LIBOMP_OMPT_SUPPORT indicates whether host OMPT support has been requested (default is ON).
+# LIBOMPTARGET_OMPT_SUPPORT indicates whether target OMPT support has been requested (default is ON).
+set(OMPT_TARGET_DEFAULT FALSE)
+if ((LIBOMP_HAVE_OMPT_SUPPORT) AND (LIBOMP_OMPT_SUPPORT) AND (NOT WIN32))
+ set (OMPT_TARGET_DEFAULT TRUE)
+endif()
+set(LIBOMPTARGET_OMPT_SUPPORT ${OMPT_TARGET_DEFAULT} CACHE BOOL "OMPT-target-support?")
+if ((OMPT_TARGET_DEFAULT) AND (LIBOMPTARGET_OMPT_SUPPORT))
+ add_definitions(-DOMPT_SUPPORT=1)
+ message(STATUS "OMPT target enabled")
+else()
+ message(STATUS "OMPT target disabled")
+endif()
+
+pythonize_bool(LIBOMPTARGET_OMPT_SUPPORT)
+
set(LIBOMPTARGET_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
+message(STATUS "OpenMP tools dir in libomptarget: ${LIBOMP_OMP_TOOLS_INCLUDE_DIR}")
+include_directories(${LIBOMP_OMP_TOOLS_INCLUDE_DIR})
# Build target agnostic offloading library.
set(LIBOMPTARGET_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src)
diff --git a/openmp/libomptarget/include/ompt_connector.h b/openmp/libomptarget/include/ompt_connector.h
new file mode 100644
index 0000000000000..f39607106d68d
--- /dev/null
+++ b/openmp/libomptarget/include/ompt_connector.h
@@ -0,0 +1,104 @@
+//=== ompt_connector.h - Target independent OpenMP target RTL -- C++ ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Support used by OMPT implementation to establish communication between
+// various OpenMP runtime libraries: host openmp library, target-independent
+// runtime library, and device-dependent runtime libraries.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _OMPT_CONNECTOR_H
+#define _OMPT_CONNECTOR_H
+
+#ifdef OMPT_SUPPORT
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <memory>
+#include <string>
+
+#include "Debug.h"
+#include "omp-tools.h"
+#include "omptarget.h"
+
+#define LIBOMPTARGET_STRINGIFY(s) #s
+
+/// Type for the function to be invoked for connecting two libraries.
+typedef void (*OmptConnectRtnTy)(ompt_start_tool_result_t *result);
+
+/// Establish connection between openmp runtime libraries
+///
+/// This class is used to communicate between an OMPT implementation in
+/// libomptarget and libomp. It is also used to communicate between an
+/// OMPT implementation in a device-specific plugin and
+/// libomptarget. The decision whether OMPT is enabled or not needs to
+/// be made when the library is loaded before any functions in the
+/// library are invoked. For that reason, an instance of this class is
+/// intended to be defined in the constructor for libomptarget or a
+/// plugin so that the decision about whether OMPT is supposed to be
+/// enabled is known before any interface function in the library is
+/// invoked.
+class OmptLibraryConnectorTy {
+public:
+ /// Use \p LibName as the prefix of the global function used for connecting
+ /// two libraries, the source indicated by \p LibName and the destination
+ /// being the one that creates this object.
+ OmptLibraryConnectorTy(const char *Ident) {
+ LibIdent.append(Ident);
+ IsInitialized = false;
+ }
+ OmptLibraryConnectorTy() = delete;
+ /// Use \p OmptResult init to connect the two libraries denoted by this
+ /// object. The init function of \p OmptResult will be used during connection
+ /// and the fini function of \p OmptResult will be used during teardown.
+ void connect(ompt_start_tool_result_t *OmptResult) {
+ initialize();
+ if (!LibConnHandle)
+ return;
+ // Call the function provided by the source library for connect
+ LibConnHandle(OmptResult);
+ }
+
+private:
+ void initialize() {
+ if (IsInitialized)
+ return;
+
+ std::string ErrMsg;
+ std::string LibName = LibIdent;
+ LibName += ".so";
+
+ DP("OMPT: Trying to load library %s\n", LibName.c_str());
+ auto DynLibHandle = std::make_unique<llvm::sys::DynamicLibrary>(
+ llvm::sys::DynamicLibrary::getPermanentLibrary(LibName.c_str(),
+ &ErrMsg));
+ if (!DynLibHandle->isValid()) {
+ // The upper layer will bail out if the handle is null.
+ LibConnHandle = nullptr;
+ } else {
+ auto LibConnRtn = "ompt_" + LibIdent + "_connect";
+ DP("OMPT: Trying to get address of connection routine %s\n",
+ LibConnRtn.c_str());
+ LibConnHandle = reinterpret_cast<OmptConnectRtnTy>(
+ DynLibHandle->getAddressOfSymbol(LibConnRtn.c_str()));
+ }
+ DP("OMPT: Library connection handle = %p\n", LibConnHandle);
+ IsInitialized = true;
+ }
+
+ /// Ensure initialization occurs only once
+ bool IsInitialized;
+ /// Handle of connect routine provided by source library
+ OmptConnectRtnTy LibConnHandle;
+ /// Name of connect routine provided by source library
+ std::string LibIdent;
+};
+
+#endif // OMPT_SUPPORT
+
+#endif // _OMPT_CONNECTOR_H
diff --git a/openmp/libomptarget/src/CMakeLists.txt b/openmp/libomptarget/src/CMakeLists.txt
index 3764f93c43ca8..2a6cd93c6ec98 100644
--- a/openmp/libomptarget/src/CMakeLists.txt
+++ b/openmp/libomptarget/src/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_library(omptarget
interface.cpp
interop.cpp
omptarget.cpp
+ ompt_callback.cpp
rtl.cpp
LegacyAPI.cpp
diff --git a/openmp/libomptarget/src/ompt_callback.cpp b/openmp/libomptarget/src/ompt_callback.cpp
new file mode 100644
index 0000000000000..4928951668caa
--- /dev/null
+++ b/openmp/libomptarget/src/ompt_callback.cpp
@@ -0,0 +1,76 @@
+//===-- ompt_callback.cpp - Target independent OpenMP target RTL -- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT callback interfaces for target independent layer
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include <assert.h>
+#include <atomic>
+#include <cstdlib>
+#include <cstring>
+
+#include "omp-tools.h"
+#include "ompt_connector.h"
+#include "private.h"
+
+#define fnptr_to_ptr(x) ((void *)(uint64_t)x)
+
+/// Used to indicate whether OMPT was enabled for this library
+bool ompt_enabled = false;
+
+/// This is the function called by the higher layer (libomp) responsible
+/// for initializing OMPT in this library. This is passed to libomp
+/// as part of the OMPT connector object.
+/// \p lookup to be used to query callbacks registered with libomp
+/// \p initial_device_num Initial device num provided by libomp
+/// \p tool_data as provided by the tool
+static int ompt_libomptarget_initialize(ompt_function_lookup_t lookup,
+ int initial_device_num,
+ ompt_data_t *tool_data) {
+ DP("enter ompt_libomptarget_initialize!\n");
+ ompt_enabled = true;
+ // TODO use the parameters to populate callbacks in libomptarget
+ DP("exit ompt_libomptarget_initialize!\n");
+ return 0;
+}
+
+static void ompt_libomptarget_finalize(ompt_data_t *data) {
+ DP("enter ompt_libomptarget_finalize!\n");
+ ompt_enabled = false;
+ DP("exit ompt_libomptarget_finalize!\n");
+}
+
+/*****************************************************************************
+ * constructor
+ *****************************************************************************/
+/// Used to initialize callbacks implemented by the tool. This interface
+/// will lookup the callbacks table in libomp and assign them to the callbacks
+/// maintained in libomptarget. Using priority 102 to have this constructor
+/// run after the init target library constructor with priority 101 (see
+/// rtl.cpp).
+__attribute__((constructor(102))) static void ompt_init(void) {
+ DP("OMPT: Enter ompt_init\n");
+ // Connect with libomp
+ static OmptLibraryConnectorTy LibompConnector("libomp");
+ static ompt_start_tool_result_t OmptResult;
+
+ // Initialize OmptResult with the init and fini functions that will be
+ // called by the connector
+ OmptResult.initialize = ompt_libomptarget_initialize;
+ OmptResult.finalize = ompt_libomptarget_finalize;
+ OmptResult.tool_data.value = 0;
+
+ // Now call connect that causes the above init/fini functions to be called
+ LibompConnector.connect(&OmptResult);
+ DP("OMPT: Exit ompt_init\n");
+}
+
+#endif // OMPT_SUPPORT
diff --git a/openmp/runtime/CMakeLists.txt b/openmp/runtime/CMakeLists.txt
index ce65a07f185fe..43299ddbad845 100644
--- a/openmp/runtime/CMakeLists.txt
+++ b/openmp/runtime/CMakeLists.txt
@@ -419,6 +419,7 @@ add_subdirectory(test)
# make these variables available for tools:
set(LIBOMP_LIBRARY_DIR ${LIBOMP_LIBRARY_DIR} PARENT_SCOPE)
set(LIBOMP_INCLUDE_DIR ${LIBOMP_INCLUDE_DIR} PARENT_SCOPE)
+set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${LIBOMP_OMP_TOOLS_INCLUDE_DIR} PARENT_SCOPE)
# make these variables available for tools/libompd:
set(LIBOMP_SRC_DIR ${LIBOMP_SRC_DIR} PARENT_SCOPE)
set(LIBOMP_OMPD_SUPPORT ${LIBOMP_OMPD_SUPPORT} PARENT_SCOPE)
diff --git a/openmp/runtime/cmake/config-ix.cmake b/openmp/runtime/cmake/config-ix.cmake
index aa79c2a605f95..f63129bd7c125 100644
--- a/openmp/runtime/cmake/config-ix.cmake
+++ b/openmp/runtime/cmake/config-ix.cmake
@@ -331,6 +331,8 @@ else()
endif()
endif()
+set(LIBOMP_HAVE_OMPT_SUPPORT ${LIBOMP_HAVE_OMPT_SUPPORT} PARENT_SCOPE)
+
# Check if HWLOC support is available
if(${LIBOMP_USE_HWLOC})
find_path(LIBOMP_HWLOC_INCLUDE_DIR NAMES hwloc.h HINTS ${LIBOMP_HWLOC_INSTALL_DIR} PATH_SUFFIXES include)
diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index d1e79b2e0c7f6..ac483b67797b0 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -391,6 +391,7 @@ if(${LIBOMP_OMPT_SUPPORT})
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH})
# install under legacy name ompt.h
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/omp-tools.h DESTINATION ${LIBOMP_HEADERS_INSTALL_PATH} RENAME ompt.h)
+ set(LIBOMP_OMP_TOOLS_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
endif()
if(${LIBOMP_FORTRAN_MODULES})
install(FILES
diff --git a/openmp/runtime/src/exports_so.txt b/openmp/runtime/src/exports_so.txt
index ac188af31055b..683f3efdf5391 100644
--- a/openmp/runtime/src/exports_so.txt
+++ b/openmp/runtime/src/exports_so.txt
@@ -26,6 +26,7 @@ VERSION {
# OMPT API
#
ompt_start_tool; # OMPT start interface
+ ompt_libomp_connect; # OMPT libomptarget interface
ompc_*; # omp.h renames some standard functions to ompc_*.
kmp_*; # Intel extensions.
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index cdafa81a18b13..55a936575b34c 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -110,6 +110,9 @@ static void *ompt_tool_module = NULL;
#define OMPT_DLCLOSE(Lib) dlclose(Lib)
#endif
+/// Used to track the initializer and the finalizer provided by libomptarget
+static ompt_start_tool_result_t *libomptarget_ompt_result = NULL;
+
/*****************************************************************************
* forward declarations
****************************************************************************/
@@ -456,7 +459,7 @@ void ompt_pre_init() {
if (verbose_init && verbose_file != stderr && verbose_file != stdout)
fclose(verbose_file);
#if OMPT_DEBUG
- printf("ompt_pre_init(): ompt_enabled = %d\n", ompt_enabled);
+ printf("ompt_pre_init(): ompt_enabled = %d\n", ompt_enabled.enabled);
#endif
}
@@ -509,12 +512,13 @@ void ompt_post_init() {
}
void ompt_fini() {
- if (ompt_enabled.enabled
-#if OMPD_SUPPORT
- && ompt_start_tool_result && ompt_start_tool_result->finalize
-#endif
- ) {
- ompt_start_tool_result->finalize(&(ompt_start_tool_result->tool_data));
+ if (ompt_enabled.enabled) {
+ if (ompt_start_tool_result && ompt_start_tool_result->finalize) {
+ ompt_start_tool_result->finalize(&(ompt_start_tool_result->tool_data));
+ }
+ if (libomptarget_ompt_result && libomptarget_ompt_result->finalize) {
+ libomptarget_ompt_result->finalize(NULL);
+ }
}
if (ompt_tool_module)
@@ -869,5 +873,50 @@ static ompt_interface_fn_t ompt_fn_lookup(const char *s) {
FOREACH_OMPT_INQUIRY_FN(ompt_interface_fn)
+#undef ompt_interface_fn
+
return NULL;
}
+
+/// Lookup function to query libomp callbacks registered by the tool
+static ompt_interface_fn_t ompt_libomp_target_fn_lookup(const char *s) {
+#define ompt_interface_fn(fn, type, code) \
+ if (strcmp(s, #fn) == 0) \
+ return (ompt_interface_fn_t)ompt_callbacks.ompt_callback(fn);
+
+ FOREACH_OMPT_DEVICE_EVENT(ompt_interface_fn)
+ FOREACH_OMPT_EMI_EVENT(ompt_interface_fn)
+ FOREACH_OMPT_NOEMI_EVENT(ompt_interface_fn)
+
+#undef ompt_interface_fn
+
+ return (ompt_interface_fn_t)0;
+}
+
+/// This function is called by the libomptarget connector to assign
+/// callbacks already registered with libomp.
+_OMP_EXTERN void ompt_libomp_connect(ompt_start_tool_result_t *result) {
+ OMPT_VERBOSE_INIT_PRINT("libomp --> OMPT: Enter libomp_ompt_connect\n");
+
+ // Ensure libomp callbacks have been added if not already
+ __ompt_force_initialization();
+
+ if (ompt_enabled.enabled &&
+ // Callbacks are initiated only if the device initialize callback
+ // has been registered by the tool
+ ompt_callbacks.ompt_callback(ompt_callback_device_initialize)) {
+ if (result) {
+ OMPT_VERBOSE_INIT_PRINT(
+ "libomp --> OMPT: Connecting with libomptarget\n");
+ // Pass in the libomp lookup function so that the already registered
+ // functions can be extracted and assigned to the callbacks in
+ // libomptarget
+ result->initialize(ompt_libomp_target_fn_lookup,
+ 0 /* initial_device_num */, nullptr /* tool_data */);
+ // Track the object provided by libomptarget so that the finalizer can be
+ // called during OMPT finalization
+ libomptarget_ompt_result = result;
+ }
+ }
+ OMPT_VERBOSE_INIT_PRINT("libomp --> OMPT: Exit libomp_ompt_connect\n");
+}
diff --git a/openmp/runtime/src/ompt-specific.cpp b/openmp/runtime/src/ompt-specific.cpp
index c28b9bd1a6605..300a403a95432 100644
--- a/openmp/runtime/src/ompt-specific.cpp
+++ b/openmp/runtime/src/ompt-specific.cpp
@@ -188,6 +188,11 @@ ompt_task_info_t *__ompt_get_scheduling_taskinfo(int depth) {
//******************************************************************************
// interface operations
//******************************************************************************
+//----------------------------------------------------------
+// initialization support
+//----------------------------------------------------------
+
+void __ompt_force_initialization() { __kmp_serial_initialize(); }
//----------------------------------------------------------
// thread support
diff --git a/openmp/runtime/src/ompt-specific.h b/openmp/runtime/src/ompt-specific.h
index bd1e0d8991e59..e68f08f45ecdb 100644
--- a/openmp/runtime/src/ompt-specific.h
+++ b/openmp/runtime/src/ompt-specific.h
@@ -20,6 +20,10 @@
* forward declarations
****************************************************************************/
+/// Entrypoint used by libomptarget to register callbacks in libomp, if not
+/// done already
+void __ompt_force_initialization();
+
void __ompt_team_assign_id(kmp_team_t *team, ompt_data_t ompt_pid);
void __ompt_thread_assign_wait_id(void *variable);
More information about the Openmp-commits
mailing list