[Openmp-commits] [PATCH] D63196: Create a runtime option to disable task throttling

[Philippe Virouleau via Phabricator via Openmp-commits]

viroulep created this revision.
viroulep added reviewers: jcownie, AndreyChurbanov.
Herald added subscribers: openmp-commits, jdoerfert.
Herald added a project: OpenMP.

By default a task execution is serialized when a task queue is full, the one exception being when its dependencies are not met, then the task queue is extended.

This behavior may cripple the application performance in some specific task graph scenarios, like the ones detailed in section 4.2 from this paper <https://link.springer.com/chapter/10.1007/978-3-319-98521-3_14> published at IWOMP 2018 (the full text can be found here <https://hal.inria.fr/hal-01901806/document>).
In such cases not having the full task graph prevent some opportunities for cache reuse between successive tasks of the stencil algorithm.

After the paper Thierry Gautier (whom I work with) was contacted by Jim Cownie who seemed to think some of the improvements described in the paper may be interesting enough to be merged upstream.

While the changes in the paper were made against an earlier version of the runtime, it's actually easier to integrate now, as the mechanism to resize task queues already exists.
So my suggestion would be to add an option to make sure the task queues are always resized if it's full and task throttling is disabled.

Please let me know what you think.


Repository:
  rOMP OpenMP

https://reviews.llvm.org/D63196

Files:
  runtime/src/kmp.h
  runtime/src/kmp_global.cpp
  runtime/src/kmp_settings.cpp
  runtime/src/kmp_tasking.cpp


Index: runtime/src/kmp_tasking.cpp
===================================================================
--- runtime/src/kmp_tasking.cpp
+++ runtime/src/kmp_tasking.cpp
@@ -374,7 +374,8 @@
   // Check if deque is full
   if (TCR_4(thread_data->td.td_deque_ntasks) >=
       TASK_DEQUE_SIZE(thread_data->td)) {
-    if (__kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
+    if (__kmp_enable_task_throttling &&
+        __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
                               thread->th.th_current_task)) {
       KA_TRACE(20, ("__kmp_push_task: T#%d deque is full; returning "
                     "TASK_NOT_PUSHED for task %p\n",
@@ -394,7 +395,8 @@
     // Need to recheck as we can get a proxy task from thread outside of OpenMP
     if (TCR_4(thread_data->td.td_deque_ntasks) >=
         TASK_DEQUE_SIZE(thread_data->td)) {
-      if (__kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
+      if (__kmp_enable_task_throttling &&
+          __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
                                 thread->th.th_current_task)) {
         __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
         KA_TRACE(20, ("__kmp_push_task: T#%d deque is full on 2nd check; "
Index: runtime/src/kmp_settings.cpp
===================================================================
--- runtime/src/kmp_settings.cpp
+++ runtime/src/kmp_settings.cpp
@@ -4682,6 +4682,20 @@
 } // __kmp_stg_print_forkjoin_frames
 #endif /* USE_ITT_BUILD */
 
+// -----------------------------------------------------------------------------
+// KMP_ENABLE_TASK_THROTTLING
+
+static void __kmp_stg_parse_task_throttling(char const *name,
+                                            char const *value, void *data) {
+  __kmp_stg_parse_bool(name, value, &__kmp_enable_task_throttling);
+} // __kmp_stg_parse_task_throttling
+
+
+static void __kmp_stg_print_task_throttling(kmp_str_buf_t *buffer,
+                                            char const *name, void *data) {
+  __kmp_stg_print_bool(buffer, name, __kmp_enable_task_throttling);
+} // __kmp_stg_print_task_throttling
+
 // -----------------------------------------------------------------------------
 // OMP_DISPLAY_ENV
 
@@ -5003,6 +5017,8 @@
     {"KMP_FORKJOIN_FRAMES_MODE", __kmp_stg_parse_forkjoin_frames_mode,
      __kmp_stg_print_forkjoin_frames_mode, NULL, 0, 0},
 #endif
+    {"KMP_ENABLE_TASK_THROTTLING", __kmp_stg_parse_task_throttling,
+     __kmp_stg_print_task_throttling, NULL, 0, 0},
 
 #if OMP_40_ENABLED
     {"OMP_DISPLAY_ENV", __kmp_stg_parse_omp_display_env,
Index: runtime/src/kmp_global.cpp
===================================================================
--- runtime/src/kmp_global.cpp
+++ runtime/src/kmp_global.cpp
@@ -341,6 +341,7 @@
 KMP_BUILD_ASSERT(sizeof(kmp_tasking_flags_t) == 4);
 
 int __kmp_task_stealing_constraint = 1; /* Constrain task stealing by default */
+int __kmp_enable_task_throttling = 1;
 
 #ifdef DEBUG_SUSPEND
 int __kmp_suspend_count = 0;
Index: runtime/src/kmp.h
===================================================================
--- runtime/src/kmp.h
+++ runtime/src/kmp.h
@@ -2119,6 +2119,7 @@
 extern kmp_tasking_mode_t
     __kmp_tasking_mode; /* determines how/when to execute tasks */
 extern int __kmp_task_stealing_constraint;
+extern int __kmp_enable_task_throttling;
 #if OMP_40_ENABLED
 extern kmp_int32 __kmp_default_device; // Set via OMP_DEFAULT_DEVICE if
 // specified, defaults to 0 otherwise


-------------- next part --------------
A non-text attachment was scrubbed...
Name: D63196.204266.patch
Type: text/x-patch
Size: 3539 bytes
Desc: not available
URL: <http://lists.llvm.org/pipermail/openmp-commits/attachments/20190612/b9c2a8dc/attachment.bin>