[Openmp-commits] [openmp] 102d864 - Fix /tmp approach, and add environment variable method as third fallback during library registration
Terry Wilmarth via Openmp-commits
openmp-commits at lists.llvm.org
Wed Sep 13 11:53:31 PDT 2023
Author: Terry Wilmarth
Date: 2023-09-13T13:50:49-05:00
New Revision: 102d864719153bb5d18a17926c536b66d258df38
URL: https://github.com/llvm/llvm-project/commit/102d864719153bb5d18a17926c536b66d258df38
DIFF: https://github.com/llvm/llvm-project/commit/102d864719153bb5d18a17926c536b66d258df38.diff
LOG: Fix /tmp approach, and add environment variable method as third fallback during library registration
The /tmp fallback for /dev/shm did not write to a fixed filename, so multiple instances of the runtime would not be able to detect each other. Now, we create the /tmp file in much the same way as the /dev/shm file was created, since mkstemp approach would not work to create a file that other instances of the runtime would detect. Also, add the environment variable method as a third fallback to /dev/shm and /tmp for library registration, as some systems do not have either. Also, add ability to fallback to a subsequent method should a failure occur during any part of the registration process. When unregistering, it is assumed that the method chosen during registration should work, so errors at that point are ignored. This also avoids a problem with multiple threads trying to unregister the library.
Added:
Modified:
openmp/runtime/src/kmp.h
openmp/runtime/src/kmp_runtime.cpp
openmp/runtime/src/z_Linux_util.cpp
Removed:
################################################################################
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index f6bfa242aaa55f3..b931b7ba66416ec 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -3620,6 +3620,9 @@ extern void __kmp_warn(char const *format, ...);
extern void __kmp_set_num_threads(int new_nth, int gtid);
+extern bool __kmp_detect_shm();
+extern bool __kmp_detect_tmp();
+
// Returns current thread (pointer to kmp_info_t). Current thread *must* be
// registered.
static inline kmp_info_t *__kmp_entry_thread() {
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 385fb6bc49cc5c7..9bf5105d134e82e 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -6717,6 +6717,8 @@ static inline char *__kmp_reg_status_name() {
} // __kmp_reg_status_get
#if defined(KMP_USE_SHM)
+bool __kmp_shm_available = false;
+bool __kmp_tmp_available = false;
// If /dev/shm is not accessible, we will create a temporary file under /tmp.
char *temp_reg_status_file_name = nullptr;
#endif
@@ -6746,60 +6748,108 @@ void __kmp_register_library_startup(void) {
char *value = NULL; // Actual value of the environment variable.
#if defined(KMP_USE_SHM)
- char *shm_name = __kmp_str_format("/%s", name);
- int shm_preexist = 0;
- char *data1;
- int fd1 = shm_open(shm_name, O_CREAT | O_EXCL | O_RDWR, 0666);
- if ((fd1 == -1) && (errno == EEXIST)) {
- // file didn't open because it already exists.
- // try opening existing file
- fd1 = shm_open(shm_name, O_RDWR, 0666);
- if (fd1 == -1) { // file didn't open
- // error out here
- __kmp_fatal(KMP_MSG(FunctionError, "Can't open SHM"), KMP_ERR(0),
- __kmp_msg_null);
- } else {
- // able to open existing file
- shm_preexist = 1;
+ char *shm_name = nullptr;
+ char *data1 = nullptr;
+ __kmp_shm_available = __kmp_detect_shm();
+ if (__kmp_shm_available) {
+ int fd1 = -1;
+ shm_name = __kmp_str_format("/%s", name);
+ int shm_preexist = 0;
+ fd1 = shm_open(shm_name, O_CREAT | O_EXCL | O_RDWR, 0666);
+ if ((fd1 == -1) && (errno == EEXIST)) {
+ // file didn't open because it already exists.
+ // try opening existing file
+ fd1 = shm_open(shm_name, O_RDWR, 0666);
+ if (fd1 == -1) { // file didn't open
+ KMP_WARNING(FunctionError, "Can't open SHM");
+ __kmp_shm_available = false;
+ } else { // able to open existing file
+ shm_preexist = 1;
+ }
+ }
+ if (__kmp_shm_available && shm_preexist == 0) { // SHM created, set size
+ if (ftruncate(fd1, SHM_SIZE) == -1) { // error occured setting size;
+ KMP_WARNING(FunctionError, "Can't set size of SHM");
+ __kmp_shm_available = false;
+ }
+ }
+ if (__kmp_shm_available) { // SHM exists, now map it
+ data1 = (char *)mmap(0, SHM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED,
+ fd1, 0);
+ if (data1 == MAP_FAILED) { // failed to map shared memory
+ KMP_WARNING(FunctionError, "Can't map SHM");
+ __kmp_shm_available = false;
+ }
+ }
+ if (__kmp_shm_available) { // SHM mapped
+ if (shm_preexist == 0) { // set data to SHM, set value
+ KMP_STRCPY_S(data1, SHM_SIZE, __kmp_registration_str);
+ }
+ // Read value from either what we just wrote or existing file.
+ value = __kmp_str_format("%s", data1); // read value from SHM
+ munmap(data1, SHM_SIZE);
}
- } else if (fd1 == -1) {
- // SHM didn't open; it was due to error other than already exists. Try to
- // create a temp file under /tmp.
+ if (fd1 != -1)
+ close(fd1);
+ }
+ if (!__kmp_shm_available)
+ __kmp_tmp_available = __kmp_detect_tmp();
+ if (!__kmp_shm_available && __kmp_tmp_available) {
+ // SHM failed to work due to an error other than that the file already
+ // exists. Try to create a temp file under /tmp.
+ // If /tmp isn't accessible, fall back to using environment variable.
// TODO: /tmp might not always be the temporary directory. For now we will
- // not consider TMPDIR. If /tmp is not accessible, we simply error out.
- char *temp_file_name = __kmp_str_format("/tmp/%sXXXXXX", name);
- fd1 = mkstemp(temp_file_name);
- if (fd1 == -1) {
- // error out here.
- __kmp_fatal(KMP_MSG(FunctionError, "Can't open TEMP"), KMP_ERR(errno),
- __kmp_msg_null);
+ // not consider TMPDIR.
+ int fd1 = -1;
+ temp_reg_status_file_name = __kmp_str_format("/tmp/%s", name);
+ int tmp_preexist = 0;
+ fd1 = open(temp_reg_status_file_name, O_CREAT | O_EXCL | O_RDWR, 0666);
+ if ((fd1 == -1) && (errno == EEXIST)) {
+ // file didn't open because it already exists.
+ // try opening existing file
+ fd1 = open(temp_reg_status_file_name, O_RDWR, 0666);
+ if (fd1 == -1) { // file didn't open if (fd1 == -1) {
+ KMP_WARNING(FunctionError, "Can't open TEMP");
+ __kmp_tmp_available = false;
+ } else {
+ tmp_preexist = 1;
+ }
}
- temp_reg_status_file_name = temp_file_name;
- }
- if (shm_preexist == 0) {
- // we created SHM now set size
- if (ftruncate(fd1, SHM_SIZE) == -1) {
- // error occured setting size;
- __kmp_fatal(KMP_MSG(FunctionError, "Can't set size of SHM"),
- KMP_ERR(errno), __kmp_msg_null);
+ if (__kmp_tmp_available && tmp_preexist == 0) {
+ // we created /tmp file now set size
+ if (ftruncate(fd1, SHM_SIZE) == -1) { // error occured setting size;
+ KMP_WARNING(FunctionError, "Can't set size of /tmp file");
+ __kmp_tmp_available = false;
+ }
}
+ if (__kmp_tmp_available) {
+ data1 = (char *)mmap(0, SHM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED,
+ fd1, 0);
+ if (data1 == MAP_FAILED) { // failed to map /tmp
+ KMP_WARNING(FunctionError, "Can't map /tmp");
+ __kmp_tmp_available = false;
+ }
+ }
+ if (__kmp_tmp_available) {
+ if (tmp_preexist == 0) { // set data to TMP, set value
+ KMP_STRCPY_S(data1, SHM_SIZE, __kmp_registration_str);
+ }
+ // Read value from either what we just wrote or existing file.
+ value = __kmp_str_format("%s", data1); // read value from SHM
+ munmap(data1, SHM_SIZE);
+ }
+ if (fd1 != -1)
+ close(fd1);
}
- data1 =
- (char *)mmap(0, SHM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd1, 0);
- if (data1 == MAP_FAILED) {
- // failed to map shared memory
- __kmp_fatal(KMP_MSG(FunctionError, "Can't map SHM"), KMP_ERR(errno),
- __kmp_msg_null);
- }
- if (shm_preexist == 0) { // set data to SHM, set value
- KMP_STRCPY_S(data1, SHM_SIZE, __kmp_registration_str);
+ if (!__kmp_shm_available && !__kmp_tmp_available) {
+ // no /dev/shm and no /tmp -- fall back to environment variable
+ // Set environment variable, but do not overwrite if it exists.
+ __kmp_env_set(name, __kmp_registration_str, 0);
+ // read value to see if it got set
+ value = __kmp_env_get(name);
}
- // Read value from either what we just wrote or existing file.
- value = __kmp_str_format("%s", data1); // read value from SHM
- munmap(data1, SHM_SIZE);
- close(fd1);
#else // Windows and unix with static library
- // Set environment variable, but do not overwrite if it is exist.
+ // Set environment variable, but do not overwrite if it exists.
__kmp_env_set(name, __kmp_registration_str, 0);
// read value to see if it got set
value = __kmp_env_get(name);
@@ -6859,8 +6909,14 @@ void __kmp_register_library_startup(void) {
case 2: { // Neighbor is dead.
#if defined(KMP_USE_SHM)
- // close shared memory.
- shm_unlink(shm_name); // this removes file in /dev/shm
+ if (__kmp_shm_available) { // close shared memory.
+ shm_unlink(shm_name); // this removes file in /dev/shm
+ } else if (__kmp_tmp_available) {
+ unlink(temp_reg_status_file_name); // this removes the temp file
+ } else {
+ // Clear the variable and try to register library again.
+ __kmp_env_unset(name);
+ }
#else
// Clear the variable and try to register library again.
__kmp_env_unset(name);
@@ -6873,7 +6929,8 @@ void __kmp_register_library_startup(void) {
}
KMP_INTERNAL_FREE((void *)value);
#if defined(KMP_USE_SHM)
- KMP_INTERNAL_FREE((void *)shm_name);
+ if (shm_name)
+ KMP_INTERNAL_FREE((void *)shm_name);
#endif
} // while
KMP_INTERNAL_FREE((void *)name);
@@ -6886,25 +6943,32 @@ void __kmp_unregister_library(void) {
char *value = NULL;
#if defined(KMP_USE_SHM)
- bool use_shm = true;
- char *shm_name = __kmp_str_format("/%s", name);
- int fd1 = shm_open(shm_name, O_RDONLY, 0666);
- if (fd1 == -1) {
- // File did not open. Try the temporary file.
- use_shm = false;
- KMP_DEBUG_ASSERT(temp_reg_status_file_name);
+ char *shm_name = nullptr;
+ int fd1;
+ if (__kmp_shm_available) {
+ shm_name = __kmp_str_format("/%s", name);
+ fd1 = shm_open(shm_name, O_RDONLY, 0666);
+ if (fd1 != -1) { // File opened successfully
+ char *data1 = (char *)mmap(0, SHM_SIZE, PROT_READ, MAP_SHARED, fd1, 0);
+ if (data1 != MAP_FAILED) {
+ value = __kmp_str_format("%s", data1); // read value from SHM
+ munmap(data1, SHM_SIZE);
+ }
+ close(fd1);
+ }
+ } else if (__kmp_tmp_available) { // try /tmp
fd1 = open(temp_reg_status_file_name, O_RDONLY);
- if (fd1 == -1) {
- // give it up now.
- return;
+ if (fd1 != -1) { // File opened successfully
+ char *data1 = (char *)mmap(0, SHM_SIZE, PROT_READ, MAP_SHARED, fd1, 0);
+ if (data1 != MAP_FAILED) {
+ value = __kmp_str_format("%s", data1); // read value from /tmp
+ munmap(data1, SHM_SIZE);
+ }
+ close(fd1);
}
+ } else { // fall back to envirable
+ value = __kmp_env_get(name);
}
- char *data1 = (char *)mmap(0, SHM_SIZE, PROT_READ, MAP_SHARED, fd1, 0);
- if (data1 != MAP_FAILED) {
- value = __kmp_str_format("%s", data1); // read value from SHM
- munmap(data1, SHM_SIZE);
- }
- close(fd1);
#else
value = __kmp_env_get(name);
#endif
@@ -6914,11 +6978,12 @@ void __kmp_unregister_library(void) {
if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
// Ok, this is our variable. Delete it.
#if defined(KMP_USE_SHM)
- if (use_shm) {
+ if (__kmp_shm_available) {
shm_unlink(shm_name); // this removes file in /dev/shm
- } else {
- KMP_DEBUG_ASSERT(temp_reg_status_file_name);
+ } else if (__kmp_tmp_available) {
unlink(temp_reg_status_file_name); // this removes the temp file
+ } else {
+ __kmp_env_unset(name);
}
#else
__kmp_env_unset(name);
@@ -6926,11 +6991,10 @@ void __kmp_unregister_library(void) {
}
#if defined(KMP_USE_SHM)
- KMP_INTERNAL_FREE(shm_name);
- if (!use_shm) {
- KMP_DEBUG_ASSERT(temp_reg_status_file_name);
+ if (shm_name)
+ KMP_INTERNAL_FREE(shm_name);
+ if (temp_reg_status_file_name)
KMP_INTERNAL_FREE(temp_reg_status_file_name);
- }
#endif
KMP_INTERNAL_FREE(__kmp_registration_str);
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index 11d9ac8dc44792f..42488eb6be5dc87 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -2744,4 +2744,28 @@ void __kmp_hidden_helper_threads_deinitz_release() {
}
#endif // KMP_OS_LINUX
+bool __kmp_detect_shm() {
+ DIR *dir = opendir("/dev/shm");
+ if (dir) { // /dev/shm exists
+ closedir(dir);
+ return true;
+ } else if (ENOENT == errno) { // /dev/shm does not exist
+ return false;
+ } else { // opendir() failed
+ return false;
+ }
+}
+
+bool __kmp_detect_tmp() {
+ DIR *dir = opendir("/tmp");
+ if (dir) { // /tmp exists
+ closedir(dir);
+ return true;
+ } else if (ENOENT == errno) { // /tmp does not exist
+ return false;
+ } else { // opendir() failed
+ return false;
+ }
+}
+
// end of file //
More information about the Openmp-commits
mailing list