[Openmp-commits] [clang] [flang] [llvm] [openmp] [OpenMP] OpenMP 6.0 taskgraph support (WIP) (PR #188765)
Julian Brown via Openmp-commits
openmp-commits at lists.llvm.org
Thu Oct 29 10:51:13 PDT 2026
https://github.com/jtb20 updated https://github.com/llvm/llvm-project/pull/188765
>From 6a3ab82dc9fd9b387a6346499f0d997313c7664c Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 27 Mar 2026 11:23:09 -0500
Subject: [PATCH 01/29] [OpenMP] Mostly remove experimental taskgraph support
from runtime
Because the changes to runtime support for taskgraph record/replay
are quite extensive, they will likely be more comprehensible if the
existing parts are removed before the new parts are added. This is the
removal part.
Pull Request: https://github.com/llvm/llvm-project/pull/194046
---
openmp/runtime/src/kmp.h | 72 ---
openmp/runtime/src/kmp_global.cpp | 12 -
openmp/runtime/src/kmp_settings.cpp | 28 --
openmp/runtime/src/kmp_taskdeps.cpp | 128 +----
openmp/runtime/src/kmp_taskdeps.h | 25 +-
openmp/runtime/src/kmp_tasking.cpp | 451 +-----------------
.../test/tasking/omp_record_replay.cpp | 48 --
.../test/tasking/omp_record_replay_deps.cpp | 63 ---
.../omp_record_replay_deps_multi_succ.cpp | 56 ---
.../tasking/omp_record_replay_multiTDGs.cpp | 76 ---
.../tasking/omp_record_replay_print_dot.cpp | 80 ----
.../tasking/omp_record_replay_taskloop.cpp | 50 --
12 files changed, 6 insertions(+), 1083 deletions(-)
delete mode 100644 openmp/runtime/test/tasking/omp_record_replay.cpp
delete mode 100644 openmp/runtime/test/tasking/omp_record_replay_deps.cpp
delete mode 100644 openmp/runtime/test/tasking/omp_record_replay_deps_multi_succ.cpp
delete mode 100644 openmp/runtime/test/tasking/omp_record_replay_multiTDGs.cpp
delete mode 100644 openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp
delete mode 100644 openmp/runtime/test/tasking/omp_record_replay_taskloop.cpp
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 9978268b52aa02..26bf4933e49e42 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2619,26 +2619,6 @@ typedef struct {
} kmp_event_t;
#if OMP_TASKGRAPH_EXPERIMENTAL
-// Initial number of allocated nodes while recording
-#define INIT_MAPSIZE 50
-
-typedef struct kmp_taskgraph_flags { /*This needs to be exactly 32 bits */
- unsigned nowait : 1;
- unsigned re_record : 1;
- unsigned reserved : 30;
-} kmp_taskgraph_flags_t;
-
-/// Represents a TDG node
-typedef struct kmp_node_info {
- kmp_task_t *task; // Pointer to the actual task
- kmp_int32 *successors; // Array of the succesors ids
- kmp_int32 nsuccessors; // Number of succesors of the node
- std::atomic<kmp_int32>
- npredecessors_counter; // Number of predessors on the fly
- kmp_int32 npredecessors; // Total number of predecessors
- kmp_int32 successors_size; // Number of allocated succesors ids
- kmp_taskdata_t *parent_task; // Parent implicit task
-} kmp_node_info_t;
/// Represent a TDG's current status
typedef enum kmp_tdg_status {
@@ -2647,43 +2627,12 @@ typedef enum kmp_tdg_status {
KMP_TDG_READY = 2
} kmp_tdg_status_t;
-/// Structure that contains a TDG
-typedef struct kmp_tdg_info {
- kmp_int32 tdg_id; // Unique idenfifier of the TDG
- kmp_taskgraph_flags_t tdg_flags; // Flags related to a TDG
- kmp_int32 map_size; // Number of allocated TDG nodes
- kmp_int32 num_roots; // Number of roots tasks int the TDG
- kmp_int32 *root_tasks; // Array of tasks identifiers that are roots
- kmp_node_info_t *record_map; // Array of TDG nodes
- kmp_tdg_status_t tdg_status =
- KMP_TDG_NONE; // Status of the TDG (recording, ready...)
- std::atomic<kmp_int32> num_tasks; // Number of TDG nodes
- kmp_bootstrap_lock_t
- graph_lock; // Protect graph attributes when updated via taskloop_recur
- // Taskloop reduction related
- void *rec_taskred_data; // Data to pass to __kmpc_task_reduction_init or
- // __kmpc_taskred_init
- kmp_int32 rec_num_taskred;
-} kmp_tdg_info_t;
-
-extern int __kmp_tdg_dot;
-extern kmp_int32 __kmp_max_tdgs;
-extern kmp_tdg_info_t **__kmp_global_tdgs;
-extern kmp_int32 __kmp_curr_tdg_idx;
-extern kmp_int32 __kmp_successors_size;
-extern std::atomic<kmp_int32> __kmp_tdg_task_id;
-extern kmp_int32 __kmp_num_tdg;
#endif
typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
/* Same fields as in the #else branch, but in reverse order */
-#if OMP_TASKGRAPH_EXPERIMENTAL
- unsigned reserved31 : 4;
- unsigned onced : 1;
-#else
unsigned reserved31 : 5;
-#endif
unsigned hidden_helper : 1;
unsigned target : 1;
unsigned native : 1;
@@ -2739,13 +2688,8 @@ typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
unsigned native : 1; /* 1==gcc-compiled task, 0==intel */
unsigned target : 1;
unsigned hidden_helper : 1; /* 1 == hidden helper task */
-#if OMP_TASKGRAPH_EXPERIMENTAL
- unsigned onced : 1; /* 1==ran once already, 0==never ran, record & replay purposes */
- unsigned reserved31 : 4; /* reserved for library use */
-#else
unsigned reserved31 : 5; /* reserved for library use */
#endif
-#endif
} kmp_tasking_flags_t;
typedef struct kmp_target_data {
@@ -2795,9 +2739,6 @@ struct kmp_taskdata { /* aligned during dynamic allocation */
ompt_task_info_t ompt_task_info;
#endif
#if OMP_TASKGRAPH_EXPERIMENTAL
- bool is_taskgraph = 0; // whether the task is within a TDG
- kmp_tdg_info_t *tdg; // used to associate task with a TDG
- kmp_int32 td_tdg_task_id; // local task id in its TDG
#endif
kmp_target_data_t td_target_data;
}; // struct kmp_taskdata
@@ -4373,20 +4314,7 @@ KMP_EXPORT void __kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid,
KMP_EXPORT void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid,
void **user_lock,
uintptr_t hint);
-
#if OMP_TASKGRAPH_EXPERIMENTAL
-// Taskgraph's Record & Replay mechanism
-// __kmp_tdg_is_recording: check whether a given TDG is recording
-// status: the tdg's current status
-static inline bool __kmp_tdg_is_recording(kmp_tdg_status_t status) {
- return status == KMP_TDG_RECORDING;
-}
-
-KMP_EXPORT kmp_int32 __kmpc_start_record_task(ident_t *loc, kmp_int32 gtid,
- kmp_int32 input_flags,
- kmp_int32 tdg_id);
-KMP_EXPORT void __kmpc_end_record_task(ident_t *loc, kmp_int32 gtid,
- kmp_int32 input_flags, kmp_int32 tdg_id);
#endif
/* Interface to fast scalable reduce methods routines */
diff --git a/openmp/runtime/src/kmp_global.cpp b/openmp/runtime/src/kmp_global.cpp
index 78ae429e3d4655..de136737640a20 100644
--- a/openmp/runtime/src/kmp_global.cpp
+++ b/openmp/runtime/src/kmp_global.cpp
@@ -554,17 +554,5 @@ int __kmp_nesting_mode = 0;
int __kmp_nesting_mode_nlevels = 1;
int *__kmp_nesting_nth_level;
-#if OMP_TASKGRAPH_EXPERIMENTAL
-// TDG record & replay
-int __kmp_tdg_dot = 0;
-kmp_int32 __kmp_max_tdgs = 100;
-kmp_tdg_info_t **__kmp_global_tdgs = NULL;
-kmp_int32 __kmp_curr_tdg_idx =
- 0; // Id of the current TDG being recorded or executed
-kmp_int32 __kmp_num_tdg = 0;
-kmp_int32 __kmp_successors_size = 10; // Initial succesor size list for
- // recording
-std::atomic<kmp_int32> __kmp_tdg_task_id = 0;
-#endif
// end of file //
diff --git a/openmp/runtime/src/kmp_settings.cpp b/openmp/runtime/src/kmp_settings.cpp
index f01231b7a168b7..139a0d6df3f2e8 100644
--- a/openmp/runtime/src/kmp_settings.cpp
+++ b/openmp/runtime/src/kmp_settings.cpp
@@ -1273,28 +1273,6 @@ static void __kmp_stg_parse_num_threads(char const *name, char const *value,
K_DIAG(1, ("__kmp_dflt_team_nth == %d\n", __kmp_dflt_team_nth));
} // __kmp_stg_parse_num_threads
-#if OMP_TASKGRAPH_EXPERIMENTAL
-static void __kmp_stg_parse_max_tdgs(char const *name, char const *value,
- void *data) {
- __kmp_stg_parse_int(name, value, 0, INT_MAX, &__kmp_max_tdgs);
-} // __kmp_stg_parse_max_tdgs
-
-static void __kmp_std_print_max_tdgs(kmp_str_buf_t *buffer, char const *name,
- void *data) {
- __kmp_stg_print_int(buffer, name, __kmp_max_tdgs);
-} // __kmp_std_print_max_tdgs
-
-static void __kmp_stg_parse_tdg_dot(char const *name, char const *value,
- void *data) {
- __kmp_stg_parse_bool(name, value, &__kmp_tdg_dot);
-} // __kmp_stg_parse_tdg_dot
-
-static void __kmp_stg_print_tdg_dot(kmp_str_buf_t *buffer, char const *name,
- void *data) {
- __kmp_stg_print_bool(buffer, name, __kmp_tdg_dot);
-} // __kmp_stg_print_tdg_dot
-#endif
-
static void __kmp_stg_parse_num_hidden_helper_threads(char const *name,
char const *value,
void *data) {
@@ -5749,12 +5727,6 @@ static kmp_setting_t __kmp_stg_table[] = {
{"LIBOMP_NUM_HIDDEN_HELPER_THREADS",
__kmp_stg_parse_num_hidden_helper_threads,
__kmp_stg_print_num_hidden_helper_threads, NULL, 0, 0},
-#if OMP_TASKGRAPH_EXPERIMENTAL
- {"KMP_MAX_TDGS", __kmp_stg_parse_max_tdgs, __kmp_std_print_max_tdgs, NULL,
- 0, 0},
- {"KMP_TDG_DOT", __kmp_stg_parse_tdg_dot, __kmp_stg_print_tdg_dot, NULL, 0,
- 0},
-#endif
#if OMPT_SUPPORT
{"OMP_TOOL", __kmp_stg_parse_omp_tool, __kmp_stg_print_omp_tool, NULL, 0,
diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index b1a0848fc722f6..6884cd144f89e5 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -222,47 +222,6 @@ static kmp_depnode_list_t *__kmp_add_node(kmp_info_t *thread,
static inline void __kmp_track_dependence(kmp_int32 gtid, kmp_depnode_t *source,
kmp_depnode_t *sink,
kmp_task_t *sink_task) {
-#if OMP_TASKGRAPH_EXPERIMENTAL
- kmp_taskdata_t *task_source = KMP_TASK_TO_TASKDATA(source->dn.task);
- kmp_taskdata_t *task_sink = KMP_TASK_TO_TASKDATA(sink_task);
- if (source->dn.task && sink_task) {
- // Not supporting dependency between two tasks that one is within the TDG
- // and the other is not
- KMP_ASSERT(task_source->is_taskgraph == task_sink->is_taskgraph);
- }
- if (task_sink->is_taskgraph &&
- __kmp_tdg_is_recording(task_sink->tdg->tdg_status)) {
- kmp_node_info_t *source_info =
- &task_sink->tdg->record_map[task_source->td_tdg_task_id];
- bool exists = false;
- for (int i = 0; i < source_info->nsuccessors; i++) {
- if (source_info->successors[i] == task_sink->td_tdg_task_id) {
- exists = true;
- break;
- }
- }
- if (!exists) {
- if (source_info->nsuccessors >= source_info->successors_size) {
- kmp_uint old_size = source_info->successors_size;
- source_info->successors_size = 2 * source_info->successors_size;
- kmp_int32 *old_succ_ids = source_info->successors;
- kmp_int32 *new_succ_ids = (kmp_int32 *)__kmp_allocate(
- source_info->successors_size * sizeof(kmp_int32));
- KMP_MEMCPY(new_succ_ids, old_succ_ids, old_size * sizeof(kmp_int32));
- source_info->successors = new_succ_ids;
- __kmp_free(old_succ_ids);
- }
-
- source_info->successors[source_info->nsuccessors] =
- task_sink->td_tdg_task_id;
- source_info->nsuccessors++;
-
- kmp_node_info_t *sink_info =
- &(task_sink->tdg->record_map[task_sink->td_tdg_task_id]);
- sink_info->npredecessors++;
- }
- }
-#endif
#ifdef KMP_SUPPORT_GRAPH_OUTPUT
kmp_taskdata_t *task_source = KMP_TASK_TO_TASKDATA(source->dn.task);
// do not use sink->dn.task as that is only filled after the dependences
@@ -311,24 +270,11 @@ __kmp_depnode_link_successor(kmp_int32 gtid, kmp_info_t *thread,
// link node as successor of list elements
for (kmp_depnode_list_t *p = plist; p; p = p->next) {
kmp_depnode_t *dep = p->node;
-#if OMP_TASKGRAPH_EXPERIMENTAL
- kmp_tdg_status tdg_status = KMP_TDG_NONE;
- if (task) {
- kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task);
- if (td->is_taskgraph)
- tdg_status = KMP_TASK_TO_TASKDATA(task)->tdg->tdg_status;
- if (__kmp_tdg_is_recording(tdg_status))
- __kmp_track_dependence(gtid, dep, node, task);
- }
-#endif
if (dep->dn.task) {
KMP_ACQUIRE_DEPNODE(gtid, dep);
if (dep->dn.task) {
if (!dep->dn.successors || dep->dn.successors->node != node) {
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (!(__kmp_tdg_is_recording(tdg_status)) && task)
-#endif
- __kmp_track_dependence(gtid, dep, node, task);
+ __kmp_track_dependence(gtid, dep, node, task);
dep->dn.successors = __kmp_add_node(thread, dep->dn.successors, node);
KA_TRACE(40, ("__kmp_process_deps: T#%d adding dependence from %p to "
"%p\n",
@@ -352,44 +298,18 @@ static inline kmp_int32 __kmp_depnode_link_successor(kmp_int32 gtid,
if (!sink)
return 0;
kmp_int32 npredecessors = 0;
-#if OMP_TASKGRAPH_EXPERIMENTAL
- kmp_tdg_status tdg_status = KMP_TDG_NONE;
- kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task);
- if (task) {
- if (td->is_taskgraph)
- tdg_status = KMP_TASK_TO_TASKDATA(task)->tdg->tdg_status;
- if (__kmp_tdg_is_recording(tdg_status) && sink->dn.task)
- __kmp_track_dependence(gtid, sink, source, task);
- }
-#endif
if (sink->dn.task) {
// synchronously add source to sink' list of successors
KMP_ACQUIRE_DEPNODE(gtid, sink);
if (sink->dn.task) {
if (!sink->dn.successors || sink->dn.successors->node != source) {
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (!(__kmp_tdg_is_recording(tdg_status)) && task)
-#endif
- __kmp_track_dependence(gtid, sink, source, task);
+ __kmp_track_dependence(gtid, sink, source, task);
sink->dn.successors = __kmp_add_node(thread, sink->dn.successors, source);
KA_TRACE(40, ("__kmp_process_deps: T#%d adding dependence from %p to "
"%p\n",
gtid, KMP_TASK_TO_TASKDATA(sink->dn.task),
KMP_TASK_TO_TASKDATA(task)));
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (__kmp_tdg_is_recording(tdg_status)) {
- kmp_taskdata_t *tdd = KMP_TASK_TO_TASKDATA(sink->dn.task);
- if (tdd->is_taskgraph) {
- if (tdd->td_flags.onced)
- // decrement npredecessors if sink->dn.task belongs to a taskgraph
- // and
- // 1) the task is reset to its initial state (by kmp_free_task) or
- // 2) the task is complete but not yet reset
- npredecessors--;
- }
- }
-#endif
- npredecessors++;
+ npredecessors++;
}
}
KMP_RELEASE_DEPNODE(gtid, sink);
@@ -694,48 +614,6 @@ kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32 gtid,
kmp_info_t *thread = __kmp_threads[gtid];
kmp_taskdata_t *current_task = thread->th.th_current_task;
-#if OMP_TASKGRAPH_EXPERIMENTAL
- // record TDG with deps
- if (new_taskdata->is_taskgraph &&
- __kmp_tdg_is_recording(new_taskdata->tdg->tdg_status)) {
- kmp_tdg_info_t *tdg = new_taskdata->tdg;
- // extend record_map if needed
- if (new_taskdata->td_tdg_task_id >= tdg->map_size) {
- __kmp_acquire_bootstrap_lock(&tdg->graph_lock);
- if (new_taskdata->td_tdg_task_id >= tdg->map_size) {
- kmp_uint old_size = tdg->map_size;
- kmp_uint new_size = old_size * 2;
- kmp_node_info_t *old_record = tdg->record_map;
- kmp_node_info_t *new_record = (kmp_node_info_t *)__kmp_allocate(
- new_size * sizeof(kmp_node_info_t));
- KMP_MEMCPY(new_record, tdg->record_map,
- old_size * sizeof(kmp_node_info_t));
- tdg->record_map = new_record;
-
- __kmp_free(old_record);
-
- for (kmp_uint i = old_size; i < new_size; i++) {
- kmp_int32 *successorsList = (kmp_int32 *)__kmp_allocate(
- __kmp_successors_size * sizeof(kmp_int32));
- new_record[i].task = nullptr;
- new_record[i].successors = successorsList;
- new_record[i].nsuccessors = 0;
- new_record[i].npredecessors = 0;
- new_record[i].successors_size = __kmp_successors_size;
- KMP_ATOMIC_ST_REL(&new_record[i].npredecessors_counter, 0);
- }
- // update the size at the end, so that we avoid other
- // threads use old_record while map_size is already updated
- tdg->map_size = new_size;
- }
- __kmp_release_bootstrap_lock(&tdg->graph_lock);
- }
- tdg->record_map[new_taskdata->td_tdg_task_id].task = new_task;
- tdg->record_map[new_taskdata->td_tdg_task_id].parent_task =
- new_taskdata->td_parent;
- KMP_ATOMIC_INC(&tdg->num_tasks);
- }
-#endif
#if OMPT_SUPPORT
if (ompt_enabled.enabled) {
if (!current_task->ompt_task_info.frame.enter_frame.ptr)
diff --git a/openmp/runtime/src/kmp_taskdeps.h b/openmp/runtime/src/kmp_taskdeps.h
index 0792baf67f162c..71e8e69d44593e 100644
--- a/openmp/runtime/src/kmp_taskdeps.h
+++ b/openmp/runtime/src/kmp_taskdeps.h
@@ -95,23 +95,6 @@ static inline void __kmp_dephash_free(kmp_info_t *thread, kmp_dephash_t *h) {
extern void __kmpc_give_task(kmp_task_t *ptask, kmp_int32 start);
static inline void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task) {
-
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (task->is_taskgraph && !(__kmp_tdg_is_recording(task->tdg->tdg_status))) {
- kmp_node_info_t *TaskInfo = &(task->tdg->record_map[task->td_tdg_task_id]);
-
- for (int i = 0; i < TaskInfo->nsuccessors; i++) {
- kmp_int32 successorNumber = TaskInfo->successors[i];
- kmp_node_info_t *successor = &(task->tdg->record_map[successorNumber]);
- kmp_int32 npredecessors = KMP_ATOMIC_DEC(&successor->npredecessors_counter) - 1;
- if (successor->task != nullptr && npredecessors == 0) {
- __kmp_omp_task(gtid, successor->task, false);
- }
- }
- return;
- }
-#endif
-
kmp_info_t *thread = __kmp_threads[gtid];
kmp_depnode_t *node = task->td_depnode;
@@ -140,12 +123,8 @@ static inline void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task) {
gtid, task));
KMP_ACQUIRE_DEPNODE(gtid, node);
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (!task->is_taskgraph ||
- (task->is_taskgraph && !__kmp_tdg_is_recording(task->tdg->tdg_status)))
-#endif
- node->dn.task =
- NULL; // mark this task as finished, so no new dependencies are generated
+ node->dn.task =
+ NULL; // mark this task as finished, so no new dependencies are generated
KMP_RELEASE_DEPNODE(gtid, node);
kmp_depnode_list_t *next;
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index db12912520a0eb..eae78848988d23 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -37,10 +37,6 @@ static void __kmp_alloc_task_deque(kmp_info_t *thread,
static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
kmp_task_team_t *task_team);
static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask);
-#if OMP_TASKGRAPH_EXPERIMENTAL
-static kmp_tdg_info_t *__kmp_find_tdg(kmp_int32 tdg_id);
-int __kmp_taskloop_task(int gtid, void *ptask);
-#endif
// returns 1 if new task is allowed to execute, 0 otherwise
// checks Task Scheduling constraint (if requested) and
@@ -70,11 +66,7 @@ static bool __kmp_task_is_allowed(int gtid, const kmp_int32 is_constrained,
}
// Check mutexinoutset dependencies, acquire locks
kmp_depnode_t *node = tasknew->td_depnode;
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (!tasknew->is_taskgraph && UNLIKELY(node && (node->dn.mtx_num_locks > 0))) {
-#else
if (UNLIKELY(node && (node->dn.mtx_num_locks > 0))) {
-#endif
for (int i = 0; i < node->dn.mtx_num_locks; ++i) {
KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL);
if (__kmp_test_lock(node->dn.mtx_locks[i], gtid))
@@ -678,33 +670,12 @@ static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata,
task->data2.priority = 0;
taskdata->td_flags.freed = 1;
-#if OMP_TASKGRAPH_EXPERIMENTAL
- // do not free tasks in taskgraph
- if (!taskdata->is_taskgraph) {
-#endif
// deallocate the taskdata and shared variable blocks associated with this task
#if USE_FAST_MEMORY
__kmp_fast_free(thread, taskdata);
#else /* ! USE_FAST_MEMORY */
__kmp_thread_free(thread, taskdata);
#endif
-#if OMP_TASKGRAPH_EXPERIMENTAL
- } else {
- taskdata->td_flags.complete = 0;
- taskdata->td_flags.started = 0;
- taskdata->td_flags.freed = 0;
- taskdata->td_flags.executing = 0;
- taskdata->td_flags.task_serial =
- (taskdata->td_parent->td_flags.final ||
- taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser);
-
- // taskdata->td_allow_completion_event.pending_events_count = 1;
- KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0);
- KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
- // start at one because counts current task and children
- KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
- }
-#endif
KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata));
}
@@ -792,10 +763,6 @@ static bool __kmp_track_children_task(kmp_taskdata_t *taskdata) {
flags.detachable == TASK_DETACHABLE || flags.hidden_helper;
ret = ret ||
KMP_ATOMIC_LD_ACQ(&taskdata->td_parent->td_incomplete_child_tasks) > 0;
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (taskdata->td_taskgroup && taskdata->is_taskgraph)
- ret = ret || KMP_ATOMIC_LD_ACQ(&taskdata->td_taskgroup->count) > 0;
-#endif
return ret;
}
@@ -815,10 +782,6 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
kmp_info_t *thread = __kmp_threads[gtid];
kmp_task_team_t *task_team =
thread->th.th_task_team; // might be NULL for serial teams...
-#if OMP_TASKGRAPH_EXPERIMENTAL
- // to avoid seg fault when we need to access taskdata->td_flags after free when using vanilla taskloop
- bool is_taskgraph;
-#endif
#if KMP_DEBUG
kmp_int32 children = 0;
#endif
@@ -828,10 +791,6 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
-#if OMP_TASKGRAPH_EXPERIMENTAL
- is_taskgraph = taskdata->is_taskgraph;
-#endif
-
if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
// untied task needs to check the counter so that the task structure is not
// freed prematurely
@@ -936,9 +895,6 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
if (completed) {
taskdata->td_flags.complete = 1; // mark the task as completed
-#if OMP_TASKGRAPH_EXPERIMENTAL
- taskdata->td_flags.onced = 1; // mark the task as ran once already
-#endif
#if OMPT_SUPPORT
// This is not a detached task, we are done here
@@ -955,11 +911,7 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
#endif
KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks);
KMP_DEBUG_ASSERT(children >= 0);
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (taskdata->td_taskgroup && !taskdata->is_taskgraph)
-#else
if (taskdata->td_taskgroup)
-#endif
KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
} else if (task_team && (task_team->tt.tt_found_proxy_tasks ||
task_team->tt.tt_hidden_helper_task_encountered)) {
@@ -998,19 +950,6 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
// KMP_DEBUG_ASSERT( resumed_task->td_flags.executing == 0 );
resumed_task->td_flags.executing = 1; // resume previous task
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (is_taskgraph && __kmp_track_children_task(taskdata) &&
- taskdata->td_taskgroup) {
- // TDG: we only release taskgroup barrier here because
- // free_task_and_ancestors will call
- // __kmp_free_task, which resets all task parameters such as
- // taskdata->started, etc. If we release the barrier earlier, these
- // parameters could be read before being reset. This is not an issue for
- // non-TDG implementation because we never reuse a task(data) structure
- KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
- }
-#endif
-
KA_TRACE(
10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",
gtid, taskdata, resumed_task));
@@ -1126,9 +1065,6 @@ void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr,
task->td_flags.executing = 1;
task->td_flags.complete = 0;
task->td_flags.freed = 0;
-#if OMP_TASKGRAPH_EXPERIMENTAL
- task->td_flags.onced = 0;
-#endif
task->td_depnode = NULL;
task->td_last_tied = task;
@@ -1172,9 +1108,6 @@ void __kmp_finish_implicit_task(kmp_info_t *thread) {
if (task->td_dephash) {
int children;
task->td_flags.complete = 1;
-#if OMP_TASKGRAPH_EXPERIMENTAL
- task->td_flags.onced = 1;
-#endif
children = KMP_ATOMIC_LD_ACQ(&task->td_incomplete_child_tasks);
kmp_tasking_flags_t flags_old = task->td_flags;
if (children == 0 && flags_old.complete == 1) {
@@ -1403,11 +1336,6 @@ kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
taskdata->td_flags.executing = 0;
taskdata->td_flags.complete = 0;
taskdata->td_flags.freed = 0;
-#if OMP_TASKGRAPH_EXPERIMENTAL
- taskdata->td_flags.onced = 0;
- taskdata->is_taskgraph = 0;
- taskdata->tdg = nullptr;
-#endif
KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
// start at one because counts current task and children
KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
@@ -1443,16 +1371,6 @@ kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
}
}
-#if OMP_TASKGRAPH_EXPERIMENTAL
- kmp_tdg_info_t *tdg = __kmp_find_tdg(__kmp_curr_tdg_idx);
- if (tdg && __kmp_tdg_is_recording(tdg->tdg_status) &&
- (task_entry != (kmp_routine_entry_t)__kmp_taskloop_task)) {
- taskdata->is_taskgraph = 1;
- taskdata->tdg = __kmp_global_tdgs[__kmp_curr_tdg_idx];
- taskdata->td_task_id = KMP_GEN_TASK_ID();
- taskdata->td_tdg_task_id = KMP_ATOMIC_INC(&__kmp_tdg_task_id);
- }
-#endif
KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
gtid, taskdata, taskdata->td_parent));
@@ -1818,53 +1736,6 @@ kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
bool serialize_immediate) {
kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (new_taskdata->is_taskgraph &&
- __kmp_tdg_is_recording(new_taskdata->tdg->tdg_status)) {
- kmp_tdg_info_t *tdg = new_taskdata->tdg;
- // extend the record_map if needed
- if (new_taskdata->td_tdg_task_id >= new_taskdata->tdg->map_size) {
- __kmp_acquire_bootstrap_lock(&tdg->graph_lock);
- // map_size could have been updated by another thread if recursive
- // taskloop
- if (new_taskdata->td_tdg_task_id >= tdg->map_size) {
- kmp_uint old_size = tdg->map_size;
- kmp_uint new_size = old_size * 2;
- kmp_node_info_t *old_record = tdg->record_map;
- kmp_node_info_t *new_record = (kmp_node_info_t *)__kmp_allocate(
- new_size * sizeof(kmp_node_info_t));
-
- KMP_MEMCPY(new_record, old_record, old_size * sizeof(kmp_node_info_t));
- tdg->record_map = new_record;
-
- __kmp_free(old_record);
-
- for (kmp_uint i = old_size; i < new_size; i++) {
- kmp_int32 *successorsList = (kmp_int32 *)__kmp_allocate(
- __kmp_successors_size * sizeof(kmp_int32));
- new_record[i].task = nullptr;
- new_record[i].successors = successorsList;
- new_record[i].nsuccessors = 0;
- new_record[i].npredecessors = 0;
- new_record[i].successors_size = __kmp_successors_size;
- KMP_ATOMIC_ST_REL(&new_record[i].npredecessors_counter, 0);
- }
- // update the size at the end, so that we avoid other
- // threads use old_record while map_size is already updated
- tdg->map_size = new_size;
- }
- __kmp_release_bootstrap_lock(&tdg->graph_lock);
- }
- // record a task
- if (tdg->record_map[new_taskdata->td_tdg_task_id].task == nullptr) {
- tdg->record_map[new_taskdata->td_tdg_task_id].task = new_task;
- tdg->record_map[new_taskdata->td_tdg_task_id].parent_task =
- new_taskdata->td_parent;
- KMP_ATOMIC_INC(&tdg->num_tasks);
- }
- }
-#endif
-
/* Should we execute the new task or queue it? For now, let's just always try
to queue it. If the queue fills up, then we'll execute it. */
if (new_taskdata->td_flags.proxy == TASK_PROXY ||
@@ -2387,17 +2258,6 @@ the reduction either does not use omp_orig object, or the omp_orig is accessible
without help of the runtime library.
*/
void *__kmpc_task_reduction_init(int gtid, int num, void *data) {
-#if OMP_TASKGRAPH_EXPERIMENTAL
- kmp_tdg_info_t *tdg = __kmp_find_tdg(__kmp_curr_tdg_idx);
- if (tdg && __kmp_tdg_is_recording(tdg->tdg_status)) {
- kmp_tdg_info_t *this_tdg = __kmp_global_tdgs[__kmp_curr_tdg_idx];
- this_tdg->rec_taskred_data =
- __kmp_allocate(sizeof(kmp_task_red_input_t) * num);
- this_tdg->rec_num_taskred = num;
- KMP_MEMCPY(this_tdg->rec_taskred_data, data,
- sizeof(kmp_task_red_input_t) * num);
- }
-#endif
return __kmp_task_reduction_init(gtid, num, (kmp_task_red_input_t *)data);
}
@@ -2414,17 +2274,6 @@ Note: this entry supposes the optional compiler-generated initializer routine
has two parameters, pointer to object to be initialized and pointer to omp_orig
*/
void *__kmpc_taskred_init(int gtid, int num, void *data) {
-#if OMP_TASKGRAPH_EXPERIMENTAL
- kmp_tdg_info_t *tdg = __kmp_find_tdg(__kmp_curr_tdg_idx);
- if (tdg && __kmp_tdg_is_recording(tdg->tdg_status)) {
- kmp_tdg_info_t *this_tdg = __kmp_global_tdgs[__kmp_curr_tdg_idx];
- this_tdg->rec_taskred_data =
- __kmp_allocate(sizeof(kmp_task_red_input_t) * num);
- this_tdg->rec_num_taskred = num;
- KMP_MEMCPY(this_tdg->rec_taskred_data, data,
- sizeof(kmp_task_red_input_t) * num);
- }
-#endif
return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data);
}
@@ -2471,18 +2320,6 @@ void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) {
kmp_int32 num;
kmp_int32 tid = thread->th.th_info.ds.ds_tid;
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if ((thread->th.th_current_task->is_taskgraph) &&
- (!__kmp_tdg_is_recording(
- __kmp_global_tdgs[__kmp_curr_tdg_idx]->tdg_status))) {
- tg = thread->th.th_current_task->td_taskgroup;
- KMP_ASSERT(tg != NULL);
- KMP_ASSERT(tg->reduce_data != NULL);
- arr = (kmp_taskred_data_t *)(tg->reduce_data);
- num = tg->reduce_num_data;
- }
-#endif
-
KMP_ASSERT(data != NULL);
while (tg != NULL) {
arr = (kmp_taskred_data_t *)(tg->reduce_data);
@@ -4249,9 +4086,6 @@ static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
taskdata->td_flags.complete = 1; // mark the task as completed
-#if OMP_TASKGRAPH_EXPERIMENTAL
- taskdata->td_flags.onced = 1;
-#endif
if (taskdata->td_taskgroup)
KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
@@ -4453,12 +4287,7 @@ void __kmp_fulfill_event(kmp_event_t *event) {
// taskloop_recur: used only when dealing with taskgraph,
// indicating whether we need to update task->td_task_id
// returns: a pointer to the allocated kmp_task_t structure (task).
-kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src
-#if OMP_TASKGRAPH_EXPERIMENTAL
- ,
- int taskloop_recur
-#endif
-) {
+kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) {
kmp_task_t *task;
kmp_taskdata_t *taskdata;
kmp_taskdata_t *taskdata_src = KMP_TASK_TO_TASKDATA(task_src);
@@ -4486,11 +4315,6 @@ kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src
task = KMP_TASKDATA_TO_TASK(taskdata);
// Initialize new task (only specific fields not affected by memcpy)
-#if OMP_TASKGRAPH_EXPERIMENTAL
- if (taskdata->is_taskgraph && !taskloop_recur &&
- __kmp_tdg_is_recording(taskdata_src->tdg->tdg_status))
- taskdata->td_tdg_task_id = KMP_ATOMIC_INC(&__kmp_tdg_task_id);
-#endif
taskdata->td_task_id = KMP_GEN_TASK_ID();
if (task->shareds != NULL) { // need setup shareds pointer
shareds_offset = (char *)task_src->shareds - (char *)taskdata_src;
@@ -4719,11 +4543,7 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
}
}
-#if OMP_TASKGRAPH_EXPERIMENTAL
- next_task = __kmp_task_dup_alloc(thread, task, /* taskloop_recur */ 0);
-#else
next_task = __kmp_task_dup_alloc(thread, task); // allocate new task
-#endif
kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task);
kmp_taskloop_bounds_t next_task_bounds =
@@ -4921,12 +4741,7 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
lb1 = ub0 + st;
// create pattern task for 2nd half of the loop
-#if OMP_TASKGRAPH_EXPERIMENTAL
- next_task = __kmp_task_dup_alloc(thread, task,
- /* taskloop_recur */ 1);
-#else
next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task
-#endif
// adjust lower bound (upper bound is not changed) for the 2nd half
*(kmp_uint64 *)((char *)next_task + lower_offset) = lb1;
if (ptask_dup != NULL) // construct firstprivates, etc.
@@ -4959,12 +4774,6 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
p->codeptr_ra = codeptr_ra;
#endif
-#if OMP_TASKGRAPH_EXPERIMENTAL
- kmp_taskdata_t *new_task_data = KMP_TASK_TO_TASKDATA(new_task);
- new_task_data->tdg = taskdata->tdg;
- new_task_data->is_taskgraph = 0;
-#endif
-
#if OMPT_SUPPORT
// schedule new task with correct return address for OMPT events
__kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra);
@@ -5003,10 +4812,6 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
#endif
__kmpc_taskgroup(loc, gtid);
}
-
-#if OMP_TASKGRAPH_EXPERIMENTAL
- KMP_ATOMIC_DEC(&__kmp_tdg_task_id);
-#endif
// =========================================================================
// calculate loop parameters
kmp_taskloop_bounds_t task_bounds(task, lb, ub);
@@ -5254,257 +5059,3 @@ bool __kmpc_omp_has_task_team(kmp_int32 gtid) {
return taskdata->td_task_team != NULL;
}
-
-#if OMP_TASKGRAPH_EXPERIMENTAL
-// __kmp_find_tdg: identify a TDG through its ID
-// tdg_id: ID of the TDG
-// returns: If a TDG corresponding to this ID is found and not
-// its initial state, return the pointer to it, otherwise nullptr
-static kmp_tdg_info_t *__kmp_find_tdg(kmp_int32 tdg_id) {
- kmp_tdg_info_t *res = nullptr;
- if (__kmp_max_tdgs == 0)
- return res;
-
- if (__kmp_global_tdgs == NULL)
- __kmp_global_tdgs = (kmp_tdg_info_t **)__kmp_allocate(
- sizeof(kmp_tdg_info_t *) * __kmp_max_tdgs);
-
- if ((__kmp_global_tdgs[tdg_id]) &&
- (__kmp_global_tdgs[tdg_id]->tdg_status != KMP_TDG_NONE))
- res = __kmp_global_tdgs[tdg_id];
- return res;
-}
-
-// __kmp_print_tdg_dot: prints the TDG to a dot file
-// tdg: ID of the TDG
-// gtid: Global Thread ID
-void __kmp_print_tdg_dot(kmp_tdg_info_t *tdg, kmp_int32 gtid) {
- kmp_int32 tdg_id = tdg->tdg_id;
- KA_TRACE(10, ("__kmp_print_tdg_dot(enter): T#%d tdg_id=%d \n", gtid, tdg_id));
-
- char file_name[20];
- sprintf(file_name, "tdg_%d.dot", tdg_id);
- kmp_safe_raii_file_t tdg_file(file_name, "w");
-
- kmp_int32 num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
- fprintf(tdg_file,
- "digraph TDG {\n"
- " compound=true\n"
- " subgraph cluster {\n"
- " label=TDG_%d\n",
- tdg_id);
- for (kmp_int32 i = 0; i < num_tasks; i++) {
- fprintf(tdg_file, " %d[style=bold]\n", i);
- }
- fprintf(tdg_file, " }\n");
- for (kmp_int32 i = 0; i < num_tasks; i++) {
- kmp_int32 nsuccessors = tdg->record_map[i].nsuccessors;
- kmp_int32 *successors = tdg->record_map[i].successors;
- if (nsuccessors > 0) {
- for (kmp_int32 j = 0; j < nsuccessors; j++)
- fprintf(tdg_file, " %d -> %d \n", i, successors[j]);
- }
- }
- fprintf(tdg_file, "}");
- KA_TRACE(10, ("__kmp_print_tdg_dot(exit): T#%d tdg_id=%d \n", gtid, tdg_id));
-}
-
-// __kmp_exec_tdg: launch the execution of a previous
-// recorded TDG
-// gtid: Global Thread ID
-// tdg: ID of the TDG
-void __kmp_exec_tdg(kmp_int32 gtid, kmp_tdg_info_t *tdg) {
- KMP_DEBUG_ASSERT(tdg->tdg_status == KMP_TDG_READY);
- KA_TRACE(10, ("__kmp_exec_tdg(enter): T#%d tdg_id=%d num_roots=%d\n", gtid,
- tdg->tdg_id, tdg->num_roots));
- kmp_node_info_t *this_record_map = tdg->record_map;
- kmp_int32 *this_root_tasks = tdg->root_tasks;
- kmp_int32 this_num_roots = tdg->num_roots;
- kmp_int32 this_num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
-
- kmp_info_t *thread = __kmp_threads[gtid];
- kmp_taskdata_t *parent_task = thread->th.th_current_task;
-
- if (tdg->rec_taskred_data) {
- __kmpc_taskred_init(gtid, tdg->rec_num_taskred, tdg->rec_taskred_data);
- }
-
- for (kmp_int32 j = 0; j < this_num_tasks; j++) {
- kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(this_record_map[j].task);
-
- td->td_parent = parent_task;
- this_record_map[j].parent_task = parent_task;
-
- kmp_taskgroup_t *parent_taskgroup =
- this_record_map[j].parent_task->td_taskgroup;
-
- KMP_ATOMIC_ST_RLX(&this_record_map[j].npredecessors_counter,
- this_record_map[j].npredecessors);
- KMP_ATOMIC_INC(&this_record_map[j].parent_task->td_incomplete_child_tasks);
-
- if (parent_taskgroup) {
- KMP_ATOMIC_INC(&parent_taskgroup->count);
- // The taskgroup is different so we must update it
- td->td_taskgroup = parent_taskgroup;
- } else if (td->td_taskgroup != nullptr) {
- // If the parent doesnt have a taskgroup, remove it from the task
- td->td_taskgroup = nullptr;
- }
- if (this_record_map[j].parent_task->td_flags.tasktype == TASK_EXPLICIT)
- KMP_ATOMIC_INC(&this_record_map[j].parent_task->td_allocated_child_tasks);
- }
-
- for (kmp_int32 j = 0; j < this_num_roots; ++j) {
- __kmp_omp_task(gtid, this_record_map[this_root_tasks[j]].task, true);
- }
- KA_TRACE(10, ("__kmp_exec_tdg(exit): T#%d tdg_id=%d num_roots=%d\n", gtid,
- tdg->tdg_id, tdg->num_roots));
-}
-
-// __kmp_start_record: set up a TDG structure and turn the
-// recording flag to true
-// gtid: Global Thread ID of the encountering thread
-// input_flags: Flags associated with the TDG
-// tdg_id: ID of the TDG to record
-static inline void __kmp_start_record(kmp_int32 gtid,
- kmp_taskgraph_flags_t *flags,
- kmp_int32 tdg_id) {
- kmp_tdg_info_t *tdg =
- (kmp_tdg_info_t *)__kmp_allocate(sizeof(kmp_tdg_info_t));
- __kmp_global_tdgs[__kmp_curr_tdg_idx] = tdg;
- // Initializing the TDG structure
- tdg->tdg_id = tdg_id;
- tdg->map_size = INIT_MAPSIZE;
- tdg->num_roots = -1;
- tdg->root_tasks = nullptr;
- tdg->tdg_status = KMP_TDG_RECORDING;
- tdg->rec_num_taskred = 0;
- tdg->rec_taskred_data = nullptr;
- KMP_ATOMIC_ST_RLX(&tdg->num_tasks, 0);
-
- // Initializing the list of nodes in this TDG
- kmp_node_info_t *this_record_map =
- (kmp_node_info_t *)__kmp_allocate(INIT_MAPSIZE * sizeof(kmp_node_info_t));
- for (kmp_int32 i = 0; i < INIT_MAPSIZE; i++) {
- kmp_int32 *successorsList =
- (kmp_int32 *)__kmp_allocate(__kmp_successors_size * sizeof(kmp_int32));
- this_record_map[i].task = nullptr;
- this_record_map[i].successors = successorsList;
- this_record_map[i].nsuccessors = 0;
- this_record_map[i].npredecessors = 0;
- this_record_map[i].successors_size = __kmp_successors_size;
- KMP_ATOMIC_ST_RLX(&this_record_map[i].npredecessors_counter, 0);
- }
-
- __kmp_global_tdgs[__kmp_curr_tdg_idx]->record_map = this_record_map;
-}
-
-// __kmpc_start_record_task: Wrapper around __kmp_start_record to mark
-// the beginning of the record process of a task region
-// loc_ref: Location of TDG, not used yet
-// gtid: Global Thread ID of the encountering thread
-// input_flags: Flags associated with the TDG
-// tdg_id: ID of the TDG to record, for now, incremental integer
-// returns: 1 if we record, otherwise, 0
-kmp_int32 __kmpc_start_record_task(ident_t *loc_ref, kmp_int32 gtid,
- kmp_int32 input_flags, kmp_int32 tdg_id) {
-
- kmp_int32 res;
- kmp_taskgraph_flags_t *flags = (kmp_taskgraph_flags_t *)&input_flags;
- KA_TRACE(10,
- ("__kmpc_start_record_task(enter): T#%d loc=%p flags=%d tdg_id=%d\n",
- gtid, loc_ref, input_flags, tdg_id));
-
- if (__kmp_max_tdgs == 0) {
- KA_TRACE(
- 10,
- ("__kmpc_start_record_task(abandon): T#%d loc=%p flags=%d tdg_id = %d, "
- "__kmp_max_tdgs = 0\n",
- gtid, loc_ref, input_flags, tdg_id));
- return 1;
- }
-
- __kmpc_taskgroup(loc_ref, gtid);
- if (kmp_tdg_info_t *tdg = __kmp_find_tdg(tdg_id)) {
- // TODO: use re_record flag
- __kmp_exec_tdg(gtid, tdg);
- res = 0;
- } else {
- __kmp_curr_tdg_idx = tdg_id;
- KMP_DEBUG_ASSERT(__kmp_curr_tdg_idx < __kmp_max_tdgs);
- __kmp_start_record(gtid, flags, tdg_id);
- __kmp_num_tdg++;
- res = 1;
- }
- KA_TRACE(10, ("__kmpc_start_record_task(exit): T#%d TDG %d starts to %s\n",
- gtid, tdg_id, res ? "record" : "execute"));
- return res;
-}
-
-// __kmp_end_record: set up a TDG after recording it
-// gtid: Global thread ID
-// tdg: Pointer to the TDG
-void __kmp_end_record(kmp_int32 gtid, kmp_tdg_info_t *tdg) {
- // Store roots
- kmp_node_info_t *this_record_map = tdg->record_map;
- kmp_int32 this_num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
- kmp_int32 *this_root_tasks =
- (kmp_int32 *)__kmp_allocate(this_num_tasks * sizeof(kmp_int32));
- kmp_int32 this_map_size = tdg->map_size;
- kmp_int32 this_num_roots = 0;
- kmp_info_t *thread = __kmp_threads[gtid];
-
- for (kmp_int32 i = 0; i < this_num_tasks; i++) {
- if (this_record_map[i].npredecessors == 0) {
- this_root_tasks[this_num_roots++] = i;
- }
- }
-
- // Update with roots info and mapsize
- tdg->map_size = this_map_size;
- tdg->num_roots = this_num_roots;
- tdg->root_tasks = this_root_tasks;
- KMP_DEBUG_ASSERT(tdg->tdg_status == KMP_TDG_RECORDING);
- tdg->tdg_status = KMP_TDG_READY;
-
- if (thread->th.th_current_task->td_dephash) {
- __kmp_dephash_free(thread, thread->th.th_current_task->td_dephash);
- thread->th.th_current_task->td_dephash = NULL;
- }
-
- // Reset predecessor counter
- for (kmp_int32 i = 0; i < this_num_tasks; i++) {
- KMP_ATOMIC_ST_RLX(&this_record_map[i].npredecessors_counter,
- this_record_map[i].npredecessors);
- }
- KMP_ATOMIC_ST_RLX(&__kmp_tdg_task_id, 0);
-
- if (__kmp_tdg_dot)
- __kmp_print_tdg_dot(tdg, gtid);
-}
-
-// __kmpc_end_record_task: wrapper around __kmp_end_record to mark
-// the end of recording phase
-//
-// loc_ref: Source location information
-// gtid: Global thread ID
-// input_flags: Flags attached to the graph
-// tdg_id: ID of the TDG just finished recording
-void __kmpc_end_record_task(ident_t *loc_ref, kmp_int32 gtid,
- kmp_int32 input_flags, kmp_int32 tdg_id) {
- kmp_tdg_info_t *tdg = __kmp_find_tdg(tdg_id);
-
- KA_TRACE(10, ("__kmpc_end_record_task(enter): T#%d loc=%p finishes recording"
- " tdg=%d with flags=%d\n",
- gtid, loc_ref, tdg_id, input_flags));
- if (__kmp_max_tdgs) {
- // TODO: use input_flags->nowait
- __kmpc_end_taskgroup(loc_ref, gtid);
- if (__kmp_tdg_is_recording(tdg->tdg_status))
- __kmp_end_record(gtid, tdg);
- }
- KA_TRACE(10, ("__kmpc_end_record_task(exit): T#%d loc=%p finished recording"
- " tdg=%d, its status is now READY\n",
- gtid, loc_ref, tdg_id));
-}
-#endif
diff --git a/openmp/runtime/test/tasking/omp_record_replay.cpp b/openmp/runtime/test/tasking/omp_record_replay.cpp
deleted file mode 100644
index 4fea22e081da97..00000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <iostream>
-#include <cassert>
-#define NT 100
-
-// Compiler-generated code (emulation)
-typedef struct ident {
- void* dummy;
-} ident_t;
-
-
-#ifdef __cplusplus
-extern "C" {
- int __kmpc_global_thread_num(ident_t *);
- int __kmpc_start_record_task(ident_t *, int, int, int);
- void __kmpc_end_record_task(ident_t *, int, int , int);
-}
-#endif
-
-void func(int *num_exec) {
- (*num_exec)++;
-}
-
-int main() {
- int num_exec = 0;
- int num_tasks = 0;
- int x=0;
- #pragma omp parallel
- #pragma omp single
- for (int iter = 0; iter < NT; ++iter) {
- int gtid = __kmpc_global_thread_num(nullptr);
- int res = __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */0);
- if (res) {
- num_tasks++;
- #pragma omp task
- func(&num_exec);
- }
- __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */0);
- }
-
- assert(num_tasks==1);
- assert(num_exec==NT);
-
- std::cout << "Passed" << std::endl;
- return 0;
-}
-// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_record_replay_deps.cpp b/openmp/runtime/test/tasking/omp_record_replay_deps.cpp
deleted file mode 100644
index 4c06ae3f7b273a..00000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay_deps.cpp
+++ /dev/null
@@ -1,63 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <iostream>
-#include <cassert>
-#define NT 100
-#define MULTIPLIER 100
-#define DECREMENT 5
-
-int val;
-// Compiler-generated code (emulation)
-typedef struct ident {
- void* dummy;
-} ident_t;
-
-
-#ifdef __cplusplus
-extern "C" {
- int __kmpc_global_thread_num(ident_t *);
- int __kmpc_start_record_task(ident_t *, int, int, int);
- void __kmpc_end_record_task(ident_t *, int, int, int);
-}
-#endif
-
-void sub() {
- #pragma omp atomic
- val -= DECREMENT;
-}
-
-void add() {
- #pragma omp atomic
- val += DECREMENT;
-}
-
-void mult() {
- // no atomicity needed, can only be executed by 1 thread
- // and no concurrency with other tasks possible
- val *= MULTIPLIER;
-}
-
-int main() {
- val = 0;
- int *x, *y;
- #pragma omp parallel
- #pragma omp single
- for (int iter = 0; iter < NT; ++iter) {
- int gtid = __kmpc_global_thread_num(nullptr);
- int res = __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */0);
- if (res) {
- #pragma omp task depend(out:y)
- add();
- #pragma omp task depend(out:x)
- sub();
- #pragma omp task depend(in:x,y)
- mult();
- }
- __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */0);
- }
- assert(val==0);
-
- std::cout << "Passed" << std::endl;
- return 0;
-}
-// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_record_replay_deps_multi_succ.cpp b/openmp/runtime/test/tasking/omp_record_replay_deps_multi_succ.cpp
deleted file mode 100644
index 6bcd3dee560300..00000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay_deps_multi_succ.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <omp.h>
-#include <cassert>
-#include <vector>
-
-constexpr const int TASKS_SIZE = 12;
-
-typedef struct ident ident_t;
-
-extern "C" {
-int __kmpc_global_thread_num(ident_t *);
-int __kmpc_start_record_task(ident_t *, int, int, int);
-void __kmpc_end_record_task(ident_t *, int, int, int);
-}
-
-void init(int &A, int val) { A = val; }
-
-void update(int &A, int &B, int val) { A = B + val; }
-
-void test(int nb, std::vector<std::vector<int>> &Ah) {
-#pragma omp parallel
-#pragma omp single
- {
- int gtid = __kmpc_global_thread_num(nullptr);
- int res = __kmpc_start_record_task(nullptr, gtid, 0, 0);
- if (res) {
- for (int k = 0; k < nb; ++k) {
-#pragma omp task depend(inout : Ah[k][0])
- init(Ah[k][0], k);
-
- for (int i = 1; i < nb; ++i) {
-#pragma omp task depend(in : Ah[k][0]) depend(out : Ah[k][i])
- update(Ah[k][i], Ah[k][0], 1);
- }
- }
- }
- __kmpc_end_record_task(nullptr, gtid, 0, 0);
- }
-}
-
-int main() {
- std::vector<std::vector<int>> matrix(TASKS_SIZE,
- std::vector<int>(TASKS_SIZE, 0));
-
- test(TASKS_SIZE, matrix);
- test(TASKS_SIZE, matrix);
-
- for (int k = 0; k < TASKS_SIZE; ++k) {
- assert(matrix[k][0] == k);
- for (int i = 1; i < TASKS_SIZE; ++i) {
- assert(matrix[k][i] == k + 1);
- }
- }
- return 0;
-}
diff --git a/openmp/runtime/test/tasking/omp_record_replay_multiTDGs.cpp b/openmp/runtime/test/tasking/omp_record_replay_multiTDGs.cpp
deleted file mode 100644
index 1864d5d89cc70e..00000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay_multiTDGs.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <iostream>
-#include <cassert>
-#define NT 20
-#define MULTIPLIER 100
-#define DECREMENT 5
-
-// Compiler-generated code (emulation)
-typedef struct ident {
- void* dummy;
-} ident_t;
-
-int val;
-#ifdef __cplusplus
-extern "C" {
- int __kmpc_global_thread_num(ident_t *);
- int __kmpc_start_record_task(ident_t *, int, int, int);
- void __kmpc_end_record_task(ident_t *, int, int , int);
-}
-#endif
-
-void sub() {
- #pragma omp atomic
- val -= DECREMENT;
-}
-
-void add() {
- #pragma omp atomic
- val += DECREMENT;
-}
-
-void mult() {
- // no atomicity needed, can only be executed by 1 thread
- // and no concurrency with other tasks possible
- val *= MULTIPLIER;
-}
-
-int main() {
- int num_tasks = 0;
- int *x, *y;
- #pragma omp parallel
- #pragma omp single
- for (int iter = 0; iter < NT; ++iter) {
- int gtid = __kmpc_global_thread_num(nullptr);
- int res = __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */0);
- if (res) {
- num_tasks++;
- #pragma omp task depend(out:y)
- add();
- #pragma omp task depend(out:x)
- sub();
- #pragma omp task depend(in:x,y)
- mult();
- }
- __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */0);
- res = __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */1);
- if (res) {
- num_tasks++;
- #pragma omp task depend(out:y)
- add();
- #pragma omp task depend(out:x)
- sub();
- #pragma omp task depend(in:x,y)
- mult();
- }
- __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */1);
- }
-
- assert(num_tasks==2);
- assert(val==0);
-
- std::cout << "Passed" << std::endl;
- return 0;
-}
-// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp b/openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp
deleted file mode 100644
index 7f1f5ccd77d37f..00000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp
+++ /dev/null
@@ -1,80 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <iostream>
-#include <fstream>
-#include <sstream>
-#include <cassert>
-
-// Compiler-generated code (emulation)
-typedef struct ident {
- void* dummy;
-} ident_t;
-
-#ifdef __cplusplus
-extern "C" {
- int __kmpc_global_thread_num(ident_t *);
- int __kmpc_start_record_task(ident_t *, int, int, int);
- void __kmpc_end_record_task(ident_t *, int, int , int);
-}
-#endif
-
-void func(int *num_exec) {
- #pragma omp atomic
- (*num_exec)++;
-}
-
-std::string tdg_string= "digraph TDG {\n"
-" compound=true\n"
-" subgraph cluster {\n"
-" label=TDG_0\n"
-" 0[style=bold]\n"
-" 1[style=bold]\n"
-" 2[style=bold]\n"
-" 3[style=bold]\n"
-" }\n"
-" 0 -> 1 \n"
-" 1 -> 2 \n"
-" 1 -> 3 \n"
-"}";
-
-int main() {
- int num_exec = 0;
- int x, y;
-
- setenv("KMP_TDG_DOT","TRUE",1);
- remove("tdg_0.dot");
-
- #pragma omp parallel
- #pragma omp single
- {
- int gtid = __kmpc_global_thread_num(nullptr);
- int res = __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */ 0);
- if (res) {
- #pragma omp task depend(out : x)
- func(&num_exec);
- #pragma omp task depend(in : x) depend(out : y)
- func(&num_exec);
- #pragma omp task depend(in : y)
- func(&num_exec);
- #pragma omp task depend(in : y)
- func(&num_exec);
- }
-
- __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */ 0);
- }
-
- assert(num_exec == 4);
-
- std::ifstream tdg_file("tdg_0.dot");
- assert(tdg_file.is_open());
-
- std::stringstream tdg_file_stream;
- tdg_file_stream << tdg_file.rdbuf();
- int equal = tdg_string.compare(tdg_file_stream.str());
-
- assert(equal == 0);
-
- std::cout << "Passed" << std::endl;
- return 0;
-}
-// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_record_replay_taskloop.cpp b/openmp/runtime/test/tasking/omp_record_replay_taskloop.cpp
deleted file mode 100644
index 163a1b4192d851..00000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay_taskloop.cpp
+++ /dev/null
@@ -1,50 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <iostream>
-#include <cassert>
-
-#define NT 20
-#define N 128*128
-
-typedef struct ident {
- void* dummy;
-} ident_t;
-
-
-#ifdef __cplusplus
-extern "C" {
- int __kmpc_global_thread_num(ident_t *);
- int __kmpc_start_record_task(ident_t *, int, int, int);
- void __kmpc_end_record_task(ident_t *, int, int , int);
-}
-#endif
-
-int main() {
- int num_tasks = 0;
-
- int array[N];
- for (int i = 0; i < N; ++i)
- array[i] = 1;
-
- long sum = 0;
- #pragma omp parallel
- #pragma omp single
- for (int iter = 0; iter < NT; ++iter) {
- int gtid = __kmpc_global_thread_num(nullptr);
- int res = __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */0);
- if (res) {
- num_tasks++;
- #pragma omp taskloop reduction(+:sum) num_tasks(4096)
- for (int i = 0; i < N; ++i) {
- sum += array[i];
- }
- }
- __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */0);
- }
- assert(sum==N*NT);
- assert(num_tasks==1);
-
- std::cout << "Passed" << std::endl;
- return 0;
-}
-// CHECK: Passed
>From 323f92a1bef01bff4c645235bdb4f75772be54fb Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 27 Mar 2026 11:56:28 -0500
Subject: [PATCH 02/29] [OpenMP] New taskgraph runtime implementation
This patch contains the bulk of the new runtime support for taskgraph
record/replay for host tasks.
Key points are as follows.
The task/taskdata structures and the dependencies between them are
duplicated whilst recording a taskgraph, keeping the existing runtime
dependency handling unaffected by the taskgraph implementation --
e.g. during runtime execution, it is valid for output dependencies to
be dropped as soon as the producing task completes. This separation
is intended to eliminate race conditions when tasks which complete
unpredictably might or might not be marked as having a subsequent task
depend on them.
The dependencies between tasks in a taskgraph are processed by static
analysis: the high-level process is akin to turning data dependencies
between tasks into control-flow dependencies. This is done by building
a set of successors and predecessors for each recorded task, then
decomposing the resulting DAG into parallel and sequential regions.
In the (presumed relatively unlikely, in real-world code) case that the
graph is irreducible, an algorithm is used to find the subgraph that
cannot be reduced, and make it an irreducible region (with children that
may be nodes, parallel or sequential regions, or even other irreducible
regions).
For irreducible regions in the output graph, the immediate children
*retain* their "intra" predecessor and successor lists. Deps pointing
in or out of the irreducible region, or those for other region types
(parallel, sequential, etc.) are dropped. Irreducible regions are
single-entry, single-exit.
To handle mutexinoutset, a fourth type of region may be used so long as
the mutual exclusion fits a restricted shape: we call that an 'exclusive'
region, and its children execute one at a time, in any order.
The output of this process is a set of nested kmp_taskgraph_region
structures -- parallel, sequential, exclusive or irreducible (each with
some number of children), or nodes representing a single task. The two
phases -- parallel/sequential decomposition and irreducible-region
handling -- alternate until we obtain a single, top-level region.
This nested structure is then used for both host-based taskgraph
replay and offload-based taskgraph replay, which later patches add.
Additional work (not yet ready for posting) has shown that it is also
maps quite well to graph execution APIs, e.g. HIP Graph.
This patch also implements host-based replay of recorded taskgraphs,
via a set of linked structures of type kmp_taskgraph_exec_descr.
This structure is built once on the first replay of a recorded graph
and then reused for subsequent replays.
A header block describing taskgraph records relating to a particular
taskgraph is located directly by using a handle passed in from the user's
compiled program, rather than using a linked list or hashtable to find
taskgraph records to replay keyed by an index: the compiler generates
a static pointer per-lexical taskgraph construct (instantiation in the
case of templated code), and the address of that is passed to the runtime
to be written back with the address of the runtime-allocated descriptor
for that construct.
Taskgraph support in this patch is mostly guarded by
OMP_TASKGRAPH_EXPERIMENTAL, except where existing code has been
shared in such a way as to make it impractical (e.g. templatizing the
dependency-handling code).
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/216324
---
openmp/runtime/src/i18n/en_US.txt | 2 +
openmp/runtime/src/kmp.h | 281 ++-
openmp/runtime/src/kmp_taskdeps.cpp | 2988 ++++++++++++++++++++++++++-
openmp/runtime/src/kmp_taskdeps.h | 27 +-
openmp/runtime/src/kmp_tasking.cpp | 1524 +++++++++++++-
5 files changed, 4675 insertions(+), 147 deletions(-)
diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index a2f6d0f6172243..d6d1b8420119c4 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -493,6 +493,8 @@ SetNumTeamsInParOrTeamsRegion "%1$s() called inside a parallel or teams region;
SetTeamsThreadLimitInParOrTeamsRegion "%1$s() called inside a parallel or teams region; call ignored."
MalformedDeviceEnvVar "%1$s: malformed device-scope environment variable name; expected `_ALL`, `_DEV`, or `_DEV_<n>` with a non-negative integer device id; ignored."
DeviceEnvVarOnGlobalScope "%1$s: device-scope environment variable suffix (`_ALL`/`_DEV`/`_DEV_<n>`) is not allowed on this base name (global-scope ICV); ignored."
+OmpTaskgraphConcurrentRecord "%1$s: multiple threads attempting to re-record taskgraph concurrently for graph_id %2$d"
+OmpTaskgraphDetachableTask "%1$s: a detachable task cannot be recorded for taskgraph replay"
# --------------------------------------------------------------------------------------------------
-*- HINTS -*-
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 26bf4933e49e42..b4697fa67f2db6 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2469,6 +2469,13 @@ extern kmp_uint64 __kmp_taskloop_min_tasks;
/*!
*/
typedef kmp_int32 (*kmp_routine_entry_t)(kmp_int32, void *);
+// A recorded task's relocation helper. At the start of a replay it makes the
+// recorded task (first argument) the one this replay generates: it re-points
+// the task's shared captures at the replay's (second argument, the taskgraph's
+// args) and constructs the initial values of the firstprivates whose variables
+// are in the taskgraph's enclosing data environment. The task's body
+// constructs its private copies from its initial values.
+typedef void (*kmp_task_relocate_t)(struct kmp_task *, void *);
typedef union kmp_cmplrdata {
kmp_int32 priority; /**< priority specified by user for the task */
@@ -2497,6 +2504,13 @@ typedef struct kmp_task { /* GEH: Shouldn't this be aligned somehow? */
@}
*/
+#if OMP_TASKGRAPH_EXPERIMENTAL
+typedef struct kmp_taskgraph_reduce_input_data {
+ void *reduce_data;
+ kmp_int32 reduce_num_data;
+} kmp_taskgraph_reduce_input_data_t;
+#endif
+
typedef struct kmp_taskgroup {
std::atomic<kmp_int32> count; // number of allocated and incomplete tasks
std::atomic<kmp_int32>
@@ -2506,6 +2520,20 @@ typedef struct kmp_taskgroup {
void *reduce_data; // reduction related info
kmp_int32 reduce_num_data; // number of data items to reduce
uintptr_t *gomp_data; // gomp reduction data
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ struct {
+ // Points to taskgraph that tasks in this taskgroup are being recorded to.
+ std::atomic<struct kmp_taskgraph_record *> recording;
+ // The task that encountered the taskgraph construct being recorded, the
+ // only one whose constructs are recorded. Set before RECORDING.
+ struct kmp_taskdata *recording_task;
+ // Temporary holding place for input data for reductions for this taskgroup
+ // during taskgraph recording. This is passed over to the first
+ // kmp_taskgraph_node we encounter inside the taskgroup. We'll have to
+ // watch out for potential race conditions here.
+ kmp_taskgraph_reduce_input_data_t *reduce_input;
+ } taskgraph;
+#endif
} kmp_taskgroup_t;
// forward declarations
@@ -2557,6 +2585,14 @@ struct kmp_depnode_list {
// Max number of mutexinoutset dependencies per node
#define MAX_MTX_DEPS 4
+#if OMP_TASKGRAPH_EXPERIMENTAL
+struct kmp_taskgraph_node;
+struct kmp_taskgraph_region;
+struct kmp_taskgraph_record;
+
+struct kmp_bitset;
+#endif
+
typedef struct kmp_base_depnode {
kmp_depnode_list_t *successors; /* used under lock */
kmp_task_t *task; /* non-NULL if depnode is active, used under lock */
@@ -2568,6 +2604,9 @@ typedef struct kmp_base_depnode {
#endif
std::atomic<kmp_int32> npredecessors;
std::atomic<kmp_int32> nrefs;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ struct kmp_bitset *set_membership;
+#endif
} kmp_base_depnode_t;
union KMP_ALIGN_CACHE kmp_depnode {
@@ -2582,7 +2621,14 @@ struct kmp_dephash_entry {
kmp_depnode_list_t *last_set;
kmp_depnode_list_t *prev_set;
kmp_uint8 last_flag;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ union {
+ kmp_lock_t *mtx_lock; /* is referenced by depnodes w/mutexinoutset dep */
+ kmp_int32 set_num;
+ };
+#else
kmp_lock_t *mtx_lock; /* is referenced by depnodes w/mutexinoutset dep */
+#endif
kmp_dephash_entry_t *next_in_bucket;
};
@@ -2620,12 +2666,191 @@ typedef struct {
#if OMP_TASKGRAPH_EXPERIMENTAL
-/// Represent a TDG's current status
-typedef enum kmp_tdg_status {
+/// Represent a task dependency graph's current status
+typedef enum kmp_taskgraph_status {
KMP_TDG_NONE = 0,
KMP_TDG_RECORDING = 1,
KMP_TDG_READY = 2
-} kmp_tdg_status_t;
+} kmp_taskgraph_status_t;
+
+enum kmp_taskgraph_mark {
+ TASKGRAPH_UNMARKED,
+ TASKGRAPH_TEMP_MARK,
+ TASKGRAPH_PERMANENT_MARK,
+ TASKGRAPH_COMBINED,
+ TASKGRAPH_DELETED
+};
+
+typedef struct kmp_taskgraph_region_dep {
+ struct kmp_taskgraph_region *region;
+ struct kmp_taskgraph_region_dep *next;
+} kmp_taskgraph_region_dep_t;
+
+enum kmp_taskgraph_region_type {
+ TASKGRAPH_REGION_ENTRY,
+ TASKGRAPH_REGION_EXIT,
+ TASKGRAPH_REGION_NODE,
+ TASKGRAPH_REGION_WAIT,
+ TASKGRAPH_REGION_PARALLEL,
+ TASKGRAPH_REGION_EXCLUSIVE,
+ TASKGRAPH_REGION_SEQUENTIAL,
+ TASKGRAPH_REGION_IRREDUCIBLE
+};
+
+typedef struct kmp_taskgraph_node {
+ kmp_task_t *task;
+ bool taskloop_task;
+ // True for undeferred (included) tasks, which block execution of the
+ // encountering task until they complete -- e.g. a taskwait without 'nowait'.
+ // Everything recorded after one is ordered after it.
+ bool undeferred;
+ kmp_taskgraph_reduce_input_data_t *reduce_input;
+ kmp_depend_info_t *dep_list;
+ kmp_int32 ndeps;
+ // This is a control dependency. If not -1, it is the index of the
+ // taskgraph node which succeeds this one in an array of taskgraph nodes.
+ kmp_int32 cfg_successor;
+} kmp_taskgraph_node_t;
+
+struct kmp_taskgraph_exec_descr_list;
+
+typedef struct kmp_taskgraph_region {
+ struct kmp_taskgraph_record *owner = nullptr;
+ // Initially, the lexical "next" region (which doesn't have to be a
+ // successor). Subsequently, a pointer to the next item in the worklist.
+ struct kmp_taskgraph_region *next = nullptr;
+ // The parent taskgraph for this one. Initially nullptr.
+ struct kmp_taskgraph_region *parent = nullptr;
+ kmp_taskgraph_region_dep_t *successors = nullptr;
+ kmp_taskgraph_region_dep_t *predecessors = nullptr;
+ // The exec descr the region is replayed through, once the exec descr array
+ // has been built.
+ struct kmp_taskgraph_exec_descr *exec_descr = nullptr;
+ // The exec descrs this region resolves to when a dependency names it, i.e.
+ // the descrs that must all complete before the region has. Only set while
+ // the enclosing IRREDUCIBLE region is being built, and only on its direct
+ // children: the members' edges point at each other, so a member's exit descrs
+ // have to stay readable until every sibling that names it has been built.
+ // This holds a *reference* on the list for that window, dropped (and the
+ // pointer cleared) once the region's members are all built.
+ struct kmp_taskgraph_exec_descr_list *exec_descrs_out = nullptr;
+ // The next allocated block.
+ struct kmp_taskgraph_region *alloc_chain;
+ struct kmp_bitset *mutexset = nullptr;
+ struct kmp_taskgraph_region *mutexset_parent = nullptr;
+ // Pointer to reduction input data for the region. We only expect to see
+ // this on TASKGRAPH_REGION_PARALLEL regions.
+ kmp_taskgraph_reduce_input_data_t *reduce_input = nullptr;
+ enum kmp_taskgraph_region_type type;
+ enum kmp_taskgraph_mark mark = TASKGRAPH_UNMARKED;
+ kmp_int32 timestamp = 0;
+ kmp_int32 level = -1;
+ union {
+ struct {
+ kmp_taskgraph_node_t *node;
+ } task;
+ struct {
+ struct kmp_taskgraph_region **children;
+ kmp_int32 num_children;
+ } inner;
+ };
+ // NODE/WAIT region.
+ kmp_taskgraph_region(struct kmp_taskgraph_record *taskgraph,
+ struct kmp_taskgraph_region **&alloc_chain,
+ kmp_taskgraph_node_t *node,
+ struct kmp_taskgraph_region *parent);
+ // PARALLEL, SEQUENTIAL, etc. with trailing children.
+ kmp_taskgraph_region(struct kmp_taskgraph_record *taskgraph,
+ struct kmp_taskgraph_region **&alloc_chain,
+ enum kmp_taskgraph_region_type type, kmp_int32 num_nodes,
+ struct kmp_taskgraph_region *parent);
+ // Fixed size allocation for leaf nodes.
+ static void *operator new(size_t size, kmp_info_p *thread);
+ // Variable sized allocation for nodes with trailing children.
+ static void *operator new(size_t size, kmp_info_p *thread,
+ kmp_int32 num_nodes);
+} kmp_taskgraph_region_t;
+
+typedef struct kmp_taskgraph_record {
+ std::atomic<kmp_taskgraph_status_t> status = KMP_TDG_NONE;
+ kmp_int32 gtid = 0;
+ uintptr_t graph_id = 0;
+ // A lock that protects the record_map and num_tasks fields from being
+ // modified by multiple threads.
+ // For now, we also use this whilst the taskgraph is being replayed.
+ // This should be replaced with an invocation counter when we implement
+ // concurrent replay of the taskgraph from different threads.
+ kmp_lock_t map_lock;
+ kmp_taskgraph_node_t *record_map = nullptr;
+ kmp_int32 num_tasks = 0;
+ kmp_int32 nodes_allocated = 0;
+ kmp_taskgraph_region_t *root = nullptr;
+ kmp_taskgraph_region_t *alloc_root = nullptr;
+ kmp_taskgraph_region_dep_t *recycled_deps = nullptr;
+ kmp_lock_t *node_mutexes = nullptr;
+ kmp_int32 num_mutexes = 0;
+ struct kmp_taskgraph_exec_descr *exec_descrs = nullptr;
+ kmp_size_t num_exec_descrs = 0;
+ // We need a taskgroup structure to keep track of recorded tasks. This is
+ // set to TRUE if the user requested "nogroup" on the taskgraph directive
+ // (then we can avoid blocking at the end of the taskgraph region on replay,
+ // at least).
+ bool nogroup_taskgroup = false;
+ // Set when this record has been reset, but has not yet been removed because
+ // it is still in use for a replay. Prevents another replay from using this
+ // record.
+ bool expired = false;
+ // Number of replays currently using this taskgraph record. At present we
+ // don't expect this to be above 1 because we don't yet support concurrent
+ // replay.
+ std::atomic<int> replay_users{0};
+ struct kmp_taskgraph_record *next = nullptr;
+} kmp_taskgraph_record_t;
+
+typedef struct kmp_taskgraph_header {
+ kmp_taskgraph_record_t *first;
+ kmp_taskgraph_record_t *expiring;
+ kmp_lock_t header_lock;
+} kmp_taskgraph_header_t;
+
+struct kmp_taskgraph_exec_descr_elem;
+
+// A node in the flattened host-replay dataflow DAG. Each task/wait gets one
+// of these, and so does a "parallel" region. The latter is where we locate
+// metadata for reductions. Other region types are represented implicitly by
+// the indegree field and successor list.
+typedef struct kmp_taskgraph_exec_descr {
+ std::atomic<kmp_int32> npredecessors;
+ // Immutable in-degree; npredecessors is reset to this at the start of every
+ // replay (the descr buffer is built once and reused).
+ kmp_int32 indegree;
+ kmp_taskgraph_region_t *region;
+ struct kmp_taskgraph_exec_descr_elem *successors;
+ // For a parallel region with reductions, the descr the region's own
+ // successors hang off. The reductions are combined into their original
+ // variables only when the region's taskgroup ends, which is after its last
+ // member completes, so the successors are released from there rather than
+ // by the members. Null for every other descr.
+ struct kmp_taskgraph_exec_descr *join;
+} kmp_taskgraph_exec_descr_t;
+
+// A single element in an exec descr list. In the exec_descr structure, these
+// are maintained in a circular list: that means that lists can be spliced
+// together in O(1) time.
+
+typedef struct kmp_taskgraph_exec_descr_elem {
+ kmp_taskgraph_exec_descr_t *exec_descr;
+ struct kmp_taskgraph_exec_descr_elem *next;
+} kmp_taskgraph_exec_descr_elem_t;
+
+// This list type is refcounted so that we don't leak memory during e.g.
+// irreducible region traversal.
+
+typedef struct kmp_taskgraph_exec_descr_list {
+ kmp_taskgraph_exec_descr_elem_t *head;
+ kmp_int32 length;
+ kmp_int32 refcount;
+} kmp_taskgraph_exec_descr_list_t;
#endif
@@ -2739,6 +2964,9 @@ struct kmp_taskdata { /* aligned during dynamic allocation */
ompt_task_info_t ompt_task_info;
#endif
#if OMP_TASKGRAPH_EXPERIMENTAL
+ // Whether the task is within a task dependency graph.
+ struct kmp_taskgraph_record *owning_taskgraph = nullptr;
+ struct kmp_taskgraph_exec_descr *exec_descr = nullptr;
#endif
kmp_target_data_t td_target_data;
}; // struct kmp_taskdata
@@ -4247,6 +4475,33 @@ KMP_EXPORT void __kmpc_omp_taskwait_deps_51(ident_t *loc_ref, kmp_int32 gtid,
extern kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
bool serialize_immediate);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+extern kmp_int32 __kmp_build_taskgraph(kmp_int32 gtid,
+ kmp_taskdata_t *current_taskdata,
+ kmp_taskgraph_record_t *taskgraph);
+extern void __kmp_region_deplist_free(kmp_info_t *thread,
+ kmp_taskgraph_region_dep_t *list);
+extern void __kmp_region_deplist_recycle(kmp_taskgraph_region_dep_t **recycled,
+ kmp_taskgraph_region_dep_t *list);
+extern kmp_int32
+__kmp_taskgraph_topological_order(kmp_taskgraph_region_t *region,
+ kmp_taskgraph_region_t **order_out,
+ kmp_int32 *outidx);
+extern void __kmp_taskgraph_acquire_locks(kmp_int32 gtid,
+ kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t *region);
+extern void __kmp_taskgraph_release_locks(kmp_int32 gtid,
+ kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t *region);
+// True if the user asked for tracing of taskgraph structure and replay via the
+// KMP_TASKGRAPH_TRACE environment variable.
+extern bool __kmp_taskgraph_trace();
+// An optional runtime verification check for taskgraph lifetime: make sure that
+// in-progress replays don't have their taskgraph record freed from underneath
+// them.
+extern bool __kmp_taskgraph_check_lifetime();
+#endif
+
KMP_EXPORT kmp_int32 __kmpc_cancel(ident_t *loc_ref, kmp_int32 gtid,
kmp_int32 cncl_kind);
KMP_EXPORT kmp_int32 __kmpc_cancellationpoint(ident_t *loc_ref, kmp_int32 gtid,
@@ -4315,6 +4570,26 @@ KMP_EXPORT void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid,
void **user_lock,
uintptr_t hint);
#if OMP_TASKGRAPH_EXPERIMENTAL
+KMP_EXPORT void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
+ std::atomic<void *> *tdg_handle,
+ uintptr_t graph_id, kmp_int32 graph_reset,
+ kmp_int32 nogroup, void (*entry)(void *),
+ void *args);
+KMP_EXPORT kmp_uint32 __kmpc_taskgraph_task(
+ ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
+ size_t sizeof_kmp_task_t, size_t sizeof_shareds, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_task_relocate_t reloc, void *task_dup);
+KMP_EXPORT kmp_uint32 __kmpc_taskgraph_taskloop(
+ ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
+ kmp_int32 if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
+ kmp_int32 nogroup, kmp_int32 sched, kmp_uint64 grainsize,
+ kmp_int32 modifier, void *task_dup, kmp_task_relocate_t reloc);
+KMP_EXPORT void __kmpc_taskgraph_taskwait(ident_t *loc_ref, kmp_int32 gtid,
+ kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list,
+ kmp_int32 has_no_wait);
+KMP_EXPORT void *__kmpc_taskgraph_taskred_init(kmp_int32 gtid, kmp_int32 num,
+ void *data);
#endif
/* Interface to fast scalable reduce methods routines */
diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index 6884cd144f89e5..9898fdea5aa338 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -20,6 +20,13 @@
#include "ompt-specific.h"
#endif
+#if OMP_TASKGRAPH_EXPERIMENTAL
+#include <bit>
+#include <cstdlib>
+#include <algorithm>
+#include <cinttypes>
+#endif
+
// TODO: Improve memory allocation? keep a list of pre-allocated structures?
// allocate in blocks? re-use list finished list entries?
// TODO: don't use atomic ref counters for stack-allocated nodes.
@@ -33,6 +40,14 @@
static std::atomic<kmp_int32> kmp_node_id_seed = 0;
#endif
+#undef DEBUG_TASKGRAPH
+
+#ifdef DEBUG_TASKGRAPH
+#define TGDBG(ARGS...) fprintf(stderr, ARGS)
+#else
+#define TGDBG(ARGS...)
+#endif
+
static void __kmp_init_node(kmp_depnode_t *node, bool on_stack) {
node->dn.successors = NULL;
node->dn.task = NULL; // will point to the right task
@@ -49,6 +64,9 @@ static void __kmp_init_node(kmp_depnode_t *node, bool on_stack) {
#ifdef KMP_SUPPORT_GRAPH_OUTPUT
node->dn.id = KMP_ATOMIC_INC(&kmp_node_id_seed);
#endif
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ node->dn.set_membership = nullptr;
+#endif
#if USE_ITT_BUILD && USE_ITT_NOTIFY
__itt_sync_create(node, "OMP task dep node", NULL, 0);
#endif
@@ -160,7 +178,12 @@ static kmp_dephash_t *__kmp_dephash_create(kmp_info_t *thread,
static kmp_dephash_entry *__kmp_dephash_find(kmp_info_t *thread,
kmp_dephash_t **hash,
- kmp_intptr_t addr) {
+ kmp_intptr_t addr
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ ,
+ bool taskgraph_p
+#endif
+) {
kmp_dephash_t *h = *hash;
if (h->nelements != 0 && h->nconflicts / h->size >= 1) {
*hash = __kmp_dephash_extend(thread, h);
@@ -190,7 +213,12 @@ static kmp_dephash_entry *__kmp_dephash_find(kmp_info_t *thread,
entry->last_set = NULL;
entry->prev_set = NULL;
entry->last_flag = 0;
- entry->mtx_lock = NULL;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (taskgraph_p)
+ entry->set_num = -1;
+ else
+#endif
+ entry->mtx_lock = NULL;
entry->next_in_bucket = h->buckets[bucket];
h->buckets[bucket] = entry;
h->nelements++;
@@ -200,6 +228,7 @@ static kmp_dephash_entry *__kmp_dephash_find(kmp_info_t *thread,
return entry;
}
+template <bool refcounting>
static kmp_depnode_list_t *__kmp_add_node(kmp_info_t *thread,
kmp_depnode_list_t *list,
kmp_depnode_t *node) {
@@ -213,7 +242,11 @@ static kmp_depnode_list_t *__kmp_add_node(kmp_info_t *thread,
thread, sizeof(kmp_depnode_list_t));
#endif
- new_head->node = __kmp_node_ref(node);
+ if (refcounting) {
+ new_head->node = __kmp_node_ref(node);
+ } else {
+ new_head->node = node;
+ }
new_head->next = list;
return new_head;
@@ -275,7 +308,8 @@ __kmp_depnode_link_successor(kmp_int32 gtid, kmp_info_t *thread,
if (dep->dn.task) {
if (!dep->dn.successors || dep->dn.successors->node != node) {
__kmp_track_dependence(gtid, dep, node, task);
- dep->dn.successors = __kmp_add_node(thread, dep->dn.successors, node);
+ dep->dn.successors =
+ __kmp_add_node<true>(thread, dep->dn.successors, node);
KA_TRACE(40, ("__kmp_process_deps: T#%d adding dependence from %p to "
"%p\n",
gtid, KMP_TASK_TO_TASKDATA(dep->dn.task),
@@ -304,7 +338,8 @@ static inline kmp_int32 __kmp_depnode_link_successor(kmp_int32 gtid,
if (sink->dn.task) {
if (!sink->dn.successors || sink->dn.successors->node != source) {
__kmp_track_dependence(gtid, sink, source, task);
- sink->dn.successors = __kmp_add_node(thread, sink->dn.successors, source);
+ sink->dn.successors =
+ __kmp_add_node<true>(thread, sink->dn.successors, source);
KA_TRACE(40, ("__kmp_process_deps: T#%d adding dependence from %p to "
"%p\n",
gtid, KMP_TASK_TO_TASKDATA(sink->dn.task),
@@ -317,21 +352,299 @@ static inline kmp_int32 __kmp_depnode_link_successor(kmp_int32 gtid,
return npredecessors;
}
+#if OMP_TASKGRAPH_EXPERIMENTAL
+kmp_taskgraph_region_dep_t *__kmp_region_deplist_add(
+ kmp_info_t *thread, kmp_taskgraph_region_dep_t **recycled_deps,
+ kmp_taskgraph_region_t *region, kmp_taskgraph_region_dep_t *list) {
+ kmp_taskgraph_region_dep_t *head;
+ if (*recycled_deps) {
+ head = *recycled_deps;
+ *recycled_deps = (*recycled_deps)->next;
+ } else
+ head = (kmp_taskgraph_region_dep_t *)__kmp_fast_allocate(
+ thread, sizeof(kmp_taskgraph_region_dep_t));
+ head->region = region;
+ head->next = list;
+ return head;
+}
+
+static bool __kmp_region_deplist_contains(kmp_taskgraph_region_dep_t *list,
+ kmp_taskgraph_region_t *region) {
+ for (; list; list = list->next)
+ if (list->region == region)
+ return true;
+ return false;
+}
+
+kmp_taskgraph_region_t *
+__kmp_region_worklist_reverse(kmp_taskgraph_region_t *list) {
+ kmp_taskgraph_region_t *last = nullptr;
+ while (list) {
+ kmp_taskgraph_region_t *next = list->next;
+ list->next = last;
+ last = list;
+ list = next;
+ }
+ return last;
+}
+
+static kmp_depnode_t *__kmp_find_in_depnode_list(kmp_depnode_t *node,
+ kmp_depnode_list_t *list) {
+ for (; list; list = list->next)
+ if (list->node == node)
+ return list->node;
+ return nullptr;
+}
+
+// A trivial fixed-size bitset implementation.
+
+typedef struct kmp_bitset {
+ kmp_uint64 *bits;
+ kmp_size_t bitsize;
+ kmp_size_t num_chunks;
+} kmp_bitset_t;
+
+static kmp_bitset_t *__kmp_bitset_alloc(kmp_info_t *thread,
+ kmp_size_t bitsize) {
+ kmp_size_t bytesize = (bitsize + 7) / 8;
+ kmp_size_t num_chunks =
+ (bytesize + sizeof(kmp_uint64) - 1) / sizeof(kmp_uint64);
+ kmp_bitset_t *bitset = (kmp_bitset_t *)__kmp_fast_allocate(
+ thread, sizeof(kmp_bitset_t) + sizeof(kmp_uint64) * num_chunks);
+ bitset->bits = (kmp_uint64 *)&bitset[1];
+ memset(bitset->bits, 0, sizeof(kmp_uint64) * num_chunks);
+ bitset->bitsize = bitsize;
+ bitset->num_chunks = num_chunks;
+ return bitset;
+}
+
+static void __kmp_bitset_free(kmp_info_t *thread, kmp_bitset_t *bitset) {
+ __kmp_fast_free(thread, bitset);
+}
+
+static void __kmp_bitset_set(kmp_bitset_t *bitset, kmp_size_t bitnum) {
+ kmp_size_t chunk = bitnum / (8 * sizeof(kmp_uint64));
+ if (bitnum < bitset->bitsize)
+ bitset->bits[chunk] |= (kmp_uint64)1 << (bitnum & 63);
+}
+
+static void __kmp_bitset_clearall(kmp_bitset_t *bitset) {
+ if (bitset)
+ memset(bitset->bits, 0, sizeof(kmp_int64) * bitset->num_chunks);
+}
+
+[[maybe_unused]] static void __kmp_bitset_setall(kmp_bitset_t *bitset) {
+ for (kmp_int32 chunk = 0; chunk < bitset->num_chunks - 1; chunk++)
+ bitset->bits[chunk] = ~(kmp_uint64)0;
+ kmp_int32 last_chunk_numbits = bitset->bitsize & 63;
+ if (last_chunk_numbits > 0) {
+ kmp_uint64 last_chunk_bits = ~((~(kmp_uint64)0) << last_chunk_numbits);
+ bitset->bits[bitset->num_chunks - 1] = last_chunk_bits;
+ }
+}
+
+static void __kmp_bitset_copy(kmp_bitset_t *dst, const kmp_bitset_t *src) {
+ assert(dst->num_chunks == src->num_chunks);
+ assert(dst->bitsize == src->bitsize);
+ memcpy(dst->bits, src->bits, sizeof(kmp_uint64) * dst->num_chunks);
+}
+
+/// Return TRUE if \c b is a subset of \c a.
+
+[[maybe_unused]] static bool __kmp_bitset_subset_p(const kmp_bitset_t *a,
+ const kmp_bitset_t *b) {
+ if (!b)
+ return true;
+ kmp_size_t chunk_max = std::max(a->num_chunks, b->num_chunks);
+ for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+ kmp_uint64 a_bits = chunk < a->num_chunks ? a->bits[chunk] : 0;
+ kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+ if ((a_bits & b_bits) != b_bits)
+ return false;
+ }
+ return true;
+}
+
+[[maybe_unused]] static void __kmp_bitset_and(kmp_bitset_t *a, kmp_bitset_t *b,
+ kmp_bitset_t *c) {
+ kmp_size_t chunk_max = std::max(b->num_chunks, c->num_chunks);
+ for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+ kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+ kmp_uint64 c_bits = chunk < c->num_chunks ? c->bits[chunk] : 0;
+ a->bits[chunk] = b_bits & c_bits;
+ }
+}
+
+static void __kmp_bitset_and_not(kmp_bitset_t *a, kmp_bitset_t *b,
+ kmp_bitset_t *c) {
+ if (!c)
+ __kmp_bitset_copy(a, b);
+ else {
+ kmp_size_t chunk_max = std::max(b->num_chunks, c->num_chunks);
+ for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+ kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+ kmp_uint64 c_bits = chunk < c->num_chunks ? c->bits[chunk] : 0;
+ a->bits[chunk] = b_bits & ~c_bits;
+ }
+ }
+}
+
+static void __kmp_bitset_or(kmp_bitset_t *a, kmp_bitset_t *b, kmp_bitset_t *c) {
+ if (!b && !c)
+ __kmp_bitset_clearall(a);
+ else if (!b)
+ __kmp_bitset_copy(a, c);
+ else if (!c)
+ __kmp_bitset_copy(a, b);
+ else {
+ kmp_size_t chunk_max = std::max(b->num_chunks, c->num_chunks);
+ for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+ kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+ kmp_uint64 c_bits = chunk < c->num_chunks ? c->bits[chunk] : 0;
+ a->bits[chunk] = b_bits | c_bits;
+ }
+ }
+}
+
+static bool __kmp_bitset_empty_p(kmp_bitset_t *bitset) {
+ if (!bitset)
+ return true;
+ for (kmp_size_t chunk = 0; chunk < bitset->num_chunks; chunk++) {
+ if (bitset->bits[chunk] != 0)
+ return false;
+ }
+ return true;
+}
+
+/// Test two bitsets for equality. Note that any unused bits at the end of the
+/// last chunk are kept as zero.
+
+static bool __kmp_bitset_equal(kmp_bitset_t *a, kmp_bitset_t *b) {
+ if (!b)
+ return __kmp_bitset_empty_p(a);
+ kmp_size_t chunk_max = std::max(a->num_chunks, b->num_chunks);
+ for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+ kmp_uint64 a_bits = chunk < a->num_chunks ? a->bits[chunk] : 0;
+ kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+ if (a_bits != b_bits)
+ return false;
+ }
+ return true;
+}
+
+/// Compute a 64-bit hash value for a bitset that is equal whenever
+/// __kmp_bitset_equal is true.
+
+static kmp_uint64 __kmp_bitset_signature(const kmp_bitset_t *bitset) {
+ kmp_uint64 sig = 0;
+ for (kmp_size_t chunk = 0; chunk < bitset->num_chunks; chunk++) {
+ kmp_uint64 bits = bitset->bits[chunk];
+ if (!bits)
+ continue;
+ // Mix the word together with its position, or a set moved wholesale from
+ // one chunk to another would fold to the same signature.
+ kmp_uint64 mixed = (bits ^ chunk) * 0x9e3779b97f4a7c15ull;
+ mixed ^= mixed >> 29;
+ mixed *= 0xbf58476d1ce4e5b9ull;
+ sig ^= mixed ^ (mixed >> 32);
+ }
+ return sig;
+}
+
+/// Pick the hash bucket for a region whose predecessor and successor sets fold
+/// to PRED_SIG and SUCC_SIG. Regions that are twins land in the same bucket
+/// because their signatures agree; unrelated regions may collide, so bucket
+/// members still have to be filtered.
+
+static kmp_uint32 __kmp_taskgraph_twin_bucket(kmp_uint64 pred_sig,
+ kmp_uint64 succ_sig,
+ kmp_uint32 mask) {
+ kmp_uint64 hash = pred_sig ^ (succ_sig * 0x9e3779b97f4a7c15ull);
+ hash ^= hash >> 32;
+ return (kmp_uint32)hash & mask;
+}
+
+static bool __kmp_bitset_intersect_p(kmp_bitset_t *a, kmp_bitset_t *b) {
+ if (!a || !b)
+ return false;
+ kmp_size_t chunk_max = std::max(a->num_chunks, b->num_chunks);
+ for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+ kmp_uint64 a_bits = chunk < a->num_chunks ? a->bits[chunk] : 0;
+ kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+ if ((a_bits & b_bits) != 0)
+ return true;
+ }
+ return false;
+}
+
+static kmp_int32 __kmp_bitset_popcount(kmp_bitset_t *bitset) {
+ if (!bitset)
+ return 0;
+ kmp_int32 accum = 0;
+ for (kmp_int32 c = 0; c < bitset->num_chunks; c++) {
+ accum += std::__popcount(bitset->bits[c]);
+ }
+ return accum;
+}
+
+static bool __kmp_bitset_test(kmp_bitset_t *bitset, kmp_size_t bitnum) {
+ if (!bitset)
+ return false;
+ kmp_size_t chunk = bitnum / (8 * sizeof(kmp_uint64));
+ if (bitnum < bitset->bitsize)
+ return (bitset->bits[chunk] & ((kmp_uint64)1 << (bitnum & 63))) != 0;
+ else
+ return false;
+}
+
+static kmp_int32 __kmp_taskgraph_add_dep(kmp_info_t *thread,
+ kmp_depnode_t *node,
+ kmp_depnode_list_t *plist) {
+ kmp_int32 npredecessors = 0;
+ for (; plist; plist = plist->next) {
+ kmp_depnode_t *dep = plist->node;
+ if (!dep->dn.successors ||
+ !__kmp_find_in_depnode_list(node, dep->dn.successors)) {
+ dep->dn.successors =
+ __kmp_add_node<false>(thread, dep->dn.successors, node);
+ npredecessors++;
+ }
+ }
+ return npredecessors;
+}
+
+static kmp_int32 __kmp_taskgraph_add_dep(kmp_info_t *thread,
+ kmp_depnode_t *source,
+ kmp_depnode_t *sink) {
+ if (!sink)
+ return 0;
+ kmp_int32 npredecessors = 0;
+ if (!sink->dn.successors || sink->dn.successors->node != source) {
+ if (!__kmp_find_in_depnode_list(source, sink->dn.successors)) {
+ sink->dn.successors =
+ __kmp_add_node<false>(thread, sink->dn.successors, source);
+ npredecessors++;
+ }
+ }
+ return npredecessors;
+}
+#endif
+
+template <typename T>
static inline kmp_int32
__kmp_process_dep_all(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t *h,
bool dep_barrier, kmp_task_t *task) {
- KA_TRACE(30, ("__kmp_process_dep_all: T#%d processing dep_all, "
+ KA_TRACE(30, ("__kmp_process_dep_all<%s>: T#%d processing dep_all, "
"dep_barrier = %d\n",
- gtid, dep_barrier));
+ T::name, gtid, dep_barrier));
kmp_info_t *thread = __kmp_threads[gtid];
kmp_int32 npredecessors = 0;
// process previous omp_all_memory node if any
- npredecessors +=
- __kmp_depnode_link_successor(gtid, thread, task, node, h->last_all);
- __kmp_node_deref(thread, h->last_all);
+ npredecessors += T::link_successor(gtid, thread, task, node, h->last_all);
+ T::deref(thread, h->last_all);
if (!dep_barrier) {
- h->last_all = __kmp_node_ref(node);
+ h->last_all = T::ref(node);
} else {
// if this is a sync point in the serial sequence, then the previous
// outputs are guaranteed to be completed after the execution of this
@@ -350,38 +663,37 @@ __kmp_process_dep_all(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t *h,
kmp_depnode_list_t *last_set = info->last_set;
kmp_depnode_list_t *prev_set = info->prev_set;
if (last_set) {
- npredecessors +=
- __kmp_depnode_link_successor(gtid, thread, task, node, last_set);
- __kmp_depnode_list_free(thread, last_set);
- __kmp_depnode_list_free(thread, prev_set);
+ npredecessors += T::link_successor(gtid, thread, task, node, last_set);
+ __kmp_depnode_list_free<T::rc>(thread, last_set);
+ __kmp_depnode_list_free<T::rc>(thread, prev_set);
info->last_set = NULL;
info->prev_set = NULL;
info->last_flag = 0; // no sets in this dephash entry
} else {
- npredecessors +=
- __kmp_depnode_link_successor(gtid, thread, task, node, last_out);
+ npredecessors += T::link_successor(gtid, thread, task, node, last_out);
}
- __kmp_node_deref(thread, last_out);
+ T::deref(thread, last_out);
if (!dep_barrier) {
- info->last_out = __kmp_node_ref(node);
+ info->last_out = T::ref(node);
} else {
info->last_out = NULL;
}
}
}
- KA_TRACE(30, ("__kmp_process_dep_all: T#%d found %d predecessors\n", gtid,
- npredecessors));
+ KA_TRACE(30, ("__kmp_process_dep_all<%s>: T#%d found %d predecessors\n",
+ T::name, gtid, npredecessors));
return npredecessors;
}
-template <bool filter>
+template <typename T>
static inline kmp_int32
__kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t **hash,
bool dep_barrier, kmp_int32 ndeps,
- kmp_depend_info_t *dep_list, kmp_task_t *task) {
- KA_TRACE(30, ("__kmp_process_deps<%d>: T#%d processing %d dependences : "
- "dep_barrier = %d\n",
- filter, gtid, ndeps, dep_barrier));
+ kmp_depend_info_t *dep_list, kmp_task_t *task,
+ kmp_int32 &next_mutex_set, bool filter = true) {
+ KA_TRACE(30, ("__kmp_process_deps<%s>: T#%d processing %d dependences : "
+ "dep_barrier = %d, filter = %d\n",
+ T::name, gtid, ndeps, dep_barrier, filter));
kmp_info_t *thread = __kmp_threads[gtid];
kmp_int32 npredecessors = 0;
@@ -391,28 +703,31 @@ __kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t **hash,
if (filter && dep->base_addr == 0)
continue; // skip filtered entries
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ kmp_dephash_entry_t *info =
+ __kmp_dephash_find(thread, hash, dep->base_addr, !T::rc);
+#else
kmp_dephash_entry_t *info =
__kmp_dephash_find(thread, hash, dep->base_addr);
+#endif
kmp_depnode_t *last_out = info->last_out;
kmp_depnode_list_t *last_set = info->last_set;
kmp_depnode_list_t *prev_set = info->prev_set;
if (dep->flags.out) { // out or inout --> clean lists if any
if (last_set) {
- npredecessors +=
- __kmp_depnode_link_successor(gtid, thread, task, node, last_set);
- __kmp_depnode_list_free(thread, last_set);
- __kmp_depnode_list_free(thread, prev_set);
+ npredecessors += T::link_successor(gtid, thread, task, node, last_set);
+ __kmp_depnode_list_free<T::rc>(thread, last_set);
+ __kmp_depnode_list_free<T::rc>(thread, prev_set);
info->last_set = NULL;
info->prev_set = NULL;
info->last_flag = 0; // no sets in this dephash entry
} else {
- npredecessors +=
- __kmp_depnode_link_successor(gtid, thread, task, node, last_out);
+ npredecessors += T::link_successor(gtid, thread, task, node, last_out);
}
- __kmp_node_deref(thread, last_out);
+ T::deref(thread, last_out);
if (!dep_barrier) {
- info->last_out = __kmp_node_ref(node);
+ info->last_out = T::ref(node);
} else {
// if this is a sync point in the serial sequence, then the previous
// outputs are guaranteed to be completed after the execution of this
@@ -423,27 +738,24 @@ __kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t **hash,
if (info->last_flag == 0 || info->last_flag == dep->flag) {
// last_set either didn't exist or of same dep kind
// link node as successor of the last_out if any
- npredecessors +=
- __kmp_depnode_link_successor(gtid, thread, task, node, last_out);
+ npredecessors += T::link_successor(gtid, thread, task, node, last_out);
// link node as successor of all nodes in the prev_set if any
- npredecessors +=
- __kmp_depnode_link_successor(gtid, thread, task, node, prev_set);
+ npredecessors += T::link_successor(gtid, thread, task, node, prev_set);
if (dep_barrier) {
// clean last_out and prev_set if any; don't touch last_set
- __kmp_node_deref(thread, last_out);
+ T::deref(thread, last_out);
info->last_out = NULL;
- __kmp_depnode_list_free(thread, prev_set);
+ __kmp_depnode_list_free<T::rc>(thread, prev_set);
info->prev_set = NULL;
}
} else { // last_set is of different dep kind, make it prev_set
// link node as successor of all nodes in the last_set
- npredecessors +=
- __kmp_depnode_link_successor(gtid, thread, task, node, last_set);
+ npredecessors += T::link_successor(gtid, thread, task, node, last_set);
// clean last_out if any
- __kmp_node_deref(thread, last_out);
+ T::deref(thread, last_out);
info->last_out = NULL;
// clean prev_set if any
- __kmp_depnode_list_free(thread, prev_set);
+ __kmp_depnode_list_free<T::rc>(thread, prev_set);
if (!dep_barrier) {
// move last_set to prev_set, new last_set will be allocated
info->prev_set = last_set;
@@ -457,62 +769,151 @@ __kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t **hash,
// 0 if last_set is empty, unchanged otherwise
if (!dep_barrier) {
info->last_flag = dep->flag; // store dep kind of the last_set
- info->last_set = __kmp_add_node(thread, info->last_set, node);
+ info->last_set = __kmp_add_node<T::rc>(thread, info->last_set, node);
}
// check if we are processing MTX dependency
if (dep->flag == KMP_DEP_MTX) {
- if (info->mtx_lock == NULL) {
- info->mtx_lock = (kmp_lock_t *)__kmp_allocate(sizeof(kmp_lock_t));
- __kmp_init_lock(info->mtx_lock);
- }
- KMP_DEBUG_ASSERT(node->dn.mtx_num_locks < MAX_MTX_DEPS);
- kmp_int32 m;
- // Save lock in node's array
- for (m = 0; m < MAX_MTX_DEPS; ++m) {
- // sort pointers in decreasing order to avoid potential livelock
- if (node->dn.mtx_locks[m] < info->mtx_lock) {
- KMP_DEBUG_ASSERT(!node->dn.mtx_locks[node->dn.mtx_num_locks]);
- for (int n = node->dn.mtx_num_locks; n > m; --n) {
- // shift right all lesser non-NULL pointers
- KMP_DEBUG_ASSERT(node->dn.mtx_locks[n - 1] != NULL);
- node->dn.mtx_locks[n] = node->dn.mtx_locks[n - 1];
- }
- node->dn.mtx_locks[m] = info->mtx_lock;
- break;
- }
- }
- KMP_DEBUG_ASSERT(m < MAX_MTX_DEPS); // must break from loop
- node->dn.mtx_num_locks++;
+ T::mutex_dep(thread, info, node, next_mutex_set);
}
}
}
- KA_TRACE(30, ("__kmp_process_deps<%d>: T#%d found %d predecessors\n", filter,
- gtid, npredecessors));
+ KA_TRACE(30,
+ ("__kmp_process_deps<%s>: T#%d found %d predecessors (filter: %d)\n",
+ T::name, gtid, npredecessors, filter));
return npredecessors;
}
-#define NO_DEP_BARRIER (false)
-#define DEP_BARRIER (true)
+struct normal_deps {
+ static constexpr char name[] = "normal";
+ static constexpr bool rc = true;
+ static kmp_int32 link_successor(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_task_t *task, kmp_depnode_t *source,
+ kmp_depnode_t *sink);
+ static kmp_int32 link_successor(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_task_t *task, kmp_depnode_t *node,
+ kmp_depnode_list_t *plist);
+ static kmp_depnode_t *ref(kmp_depnode_t *node);
+ static void deref(kmp_info_t *thread, kmp_depnode_t *node);
+ static void mutex_dep(kmp_info_t *thread, kmp_dephash_entry_t *info,
+ kmp_depnode_t *node, kmp_int32 &next_mutex_set);
+};
+
+kmp_int32 normal_deps::link_successor(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_task_t *task, kmp_depnode_t *source,
+ kmp_depnode_t *sink) {
+ return __kmp_depnode_link_successor(gtid, thread, task, source, sink);
+}
-// returns true if the task has any outstanding dependence
-static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
- kmp_task_t *task, kmp_dephash_t **hash,
- bool dep_barrier, kmp_int32 ndeps,
- kmp_depend_info_t *dep_list,
- kmp_int32 ndeps_noalias,
- kmp_depend_info_t *noalias_dep_list) {
- int i, n_mtxs = 0, dep_all = 0;
-#if KMP_DEBUG
- kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+kmp_int32 normal_deps::link_successor(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_task_t *task, kmp_depnode_t *node,
+ kmp_depnode_list_t *plist) {
+ return __kmp_depnode_link_successor(gtid, thread, task, node, plist);
+}
+
+kmp_depnode_t *normal_deps::ref(kmp_depnode_t *node) {
+ return __kmp_node_ref(node);
+}
+
+void normal_deps::deref(kmp_info_t *thread, kmp_depnode_t *node) {
+ __kmp_node_deref(thread, node);
+}
+
+void normal_deps::mutex_dep(kmp_info_t *thread, kmp_dephash_entry_t *info,
+ kmp_depnode_t *node, kmp_int32 &next_mutex_set) {
+ if (info->mtx_lock == NULL) {
+ info->mtx_lock = (kmp_lock_t *)__kmp_allocate(sizeof(kmp_lock_t));
+ __kmp_init_lock(info->mtx_lock);
+ }
+ KMP_DEBUG_ASSERT(node->dn.mtx_num_locks < MAX_MTX_DEPS);
+ kmp_int32 m;
+ // Save lock in node's array
+ for (m = 0; m < MAX_MTX_DEPS; ++m) {
+ // sort pointers in decreasing order to avoid potential livelock
+ if (node->dn.mtx_locks[m] < info->mtx_lock) {
+ KMP_DEBUG_ASSERT(!node->dn.mtx_locks[node->dn.mtx_num_locks]);
+ for (int n = node->dn.mtx_num_locks; n > m; --n) {
+ // shift right all lesser non-NULL pointers
+ KMP_DEBUG_ASSERT(node->dn.mtx_locks[n - 1] != NULL);
+ node->dn.mtx_locks[n] = node->dn.mtx_locks[n - 1];
+ }
+ node->dn.mtx_locks[m] = info->mtx_lock;
+ break;
+ }
+ }
+ KMP_DEBUG_ASSERT(m < MAX_MTX_DEPS); // must break from loop
+ node->dn.mtx_num_locks++;
+}
+
+#if OMP_TASKGRAPH_EXPERIMENTAL
+struct taskgraph_deps {
+ static constexpr char name[] = "taskgraph";
+ static constexpr bool rc = false;
+ static kmp_int32 link_successor(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_task_t *task, kmp_depnode_t *source,
+ kmp_depnode_t *sink);
+ static kmp_int32 link_successor(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_task_t *task, kmp_depnode_t *node,
+ kmp_depnode_list_t *plist);
+ static kmp_depnode_t *ref(kmp_depnode_t *node) { return node; }
+ static void deref(kmp_info_t *thread, kmp_depnode_t *node) {}
+ static void mutex_dep(kmp_info_t *thread, kmp_dephash_entry_t *info,
+ kmp_depnode_t *node, kmp_int32 &next_mutex_set);
+};
+
+kmp_int32 taskgraph_deps::link_successor(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_task_t *task,
+ kmp_depnode_t *source,
+ kmp_depnode_t *sink) {
+ return __kmp_taskgraph_add_dep(thread, source, sink);
+}
+
+kmp_int32 taskgraph_deps::link_successor(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_task_t *task, kmp_depnode_t *node,
+ kmp_depnode_list_t *plist) {
+ return __kmp_taskgraph_add_dep(thread, node, plist);
+}
+
+void taskgraph_deps::mutex_dep(kmp_info_t *thread, kmp_dephash_entry_t *info,
+ kmp_depnode_t *node, kmp_int32 &next_mutex_set) {
+ if (info->set_num == -1) {
+ info->set_num = next_mutex_set++;
+ }
+ if (!node->dn.set_membership) {
+ node->dn.set_membership = __kmp_bitset_alloc(thread, 64);
+ }
+ __kmp_bitset_set(node->dn.set_membership, info->set_num);
+}
+
+void __kmp_taskgraph_acquire_locks(kmp_int32 gtid,
+ kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t *region) {
+ for (kmp_int32 i = 0; i < taskgraph->num_mutexes; i++) {
+ if (__kmp_bitset_test(region->mutexset, i))
+ __kmp_acquire_lock(&taskgraph->node_mutexes[i], gtid);
+ }
+}
+
+void __kmp_taskgraph_release_locks(kmp_int32 gtid,
+ kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t *region) {
+ for (kmp_int32 i = 0; i < taskgraph->num_mutexes; i++) {
+ if (__kmp_bitset_test(region->mutexset, i))
+ __kmp_release_lock(&taskgraph->node_mutexes[i], gtid);
+ }
+}
#endif
- KA_TRACE(20, ("__kmp_check_deps: T#%d checking dependences for task %p : %d "
- "possibly aliased dependences, %d non-aliased dependences : "
- "dep_barrier=%d .\n",
- gtid, taskdata, ndeps, ndeps_noalias, dep_barrier));
+
+/// Search for aliased (same base address) dependencies in \c dep_list, and
+/// nullify duplicates. Return TRUE if we have an 'all' dependency, FALSE
+/// otherwise. Return number of mutex dependencies in *N_MTXS.
+static bool __kmp_filter_aliased_deps(kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list,
+ kmp_task_t *task, int *n_mtxs) {
+ *n_mtxs = 0;
// Filter deps in dep_list
// TODO: Different algorithm for large dep_list ( > 10 ? )
- for (i = 0; i < ndeps; i++) {
+ for (int i = 0; i < ndeps; i++) {
if (dep_list[i].base_addr != 0 &&
dep_list[i].base_addr != (kmp_intptr_t)KMP_SIZE_T_MAX) {
KMP_DEBUG_ASSERT(
@@ -530,8 +931,8 @@ static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
}
if (dep_list[i].flag == KMP_DEP_MTX) {
// limit number of mtx deps to MAX_MTX_DEPS per node
- if (n_mtxs < MAX_MTX_DEPS && task != NULL) {
- ++n_mtxs;
+ if (*n_mtxs < MAX_MTX_DEPS && task != NULL) {
+ ++(*n_mtxs);
} else {
dep_list[i].flag = KMP_DEP_OUT; // downgrade mutexinoutset to inout
}
@@ -541,11 +942,2407 @@ static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
// omp_all_memory dependence can be marked by compiler by either
// (addr=0 && flag=0x80) (flag KMP_DEP_ALL), or (addr=-1).
// omp_all_memory overrides all other dependences if any
- dep_all = 1;
+ return true;
+ }
+ }
+ return false;
+}
+
+#if OMP_TASKGRAPH_EXPERIMENTAL
+// Round up a size to a power of two specified by val: Used to insert padding
+// between structures co-allocated using a single malloc() call
+// FIXME: We copy+pasted this, put it somewhere else instead.
+static size_t __kmp_round_up_to_val(size_t size, size_t val) {
+ if (size & (val - 1)) {
+ size &= ~(val - 1);
+ if (size <= KMP_SIZE_T_MAX - val) {
+ size += val; // Round up if there is no overflow.
+ }
+ }
+ return size;
+} // __kmp_round_up_to_val
+
+void *kmp_taskgraph_region::operator new(size_t size, kmp_info_t *thread) {
+ return __kmp_fast_allocate(thread, size);
+}
+
+void *kmp_taskgraph_region::operator new(size_t size, kmp_info_t *thread,
+ kmp_int32 num_nodes) {
+ size += num_nodes * sizeof(kmp_taskgraph_region_t *);
+ size = __kmp_round_up_to_val(size, sizeof(kmp_taskgraph_region_t *));
+ return __kmp_fast_allocate(thread, size);
+}
+
+kmp_taskgraph_region::kmp_taskgraph_region(
+ kmp_taskgraph_record_t *taskgraph, kmp_taskgraph_region_t **&alloc_chain,
+ kmp_taskgraph_node_t *node, kmp_taskgraph_region_t *parent) {
+ owner = taskgraph;
+ type = node ? TASKGRAPH_REGION_NODE : TASKGRAPH_REGION_WAIT;
+ task.node = node;
+ this->parent = parent;
+ *alloc_chain = this;
+ alloc_chain = &this->alloc_chain;
+}
+
+kmp_taskgraph_region::kmp_taskgraph_region(
+ kmp_taskgraph_record_t *taskgraph, kmp_taskgraph_region_t **&alloc_chain,
+ enum kmp_taskgraph_region_type type, kmp_int32 num_nodes,
+ kmp_taskgraph_region_t *parent) {
+ owner = taskgraph;
+ this->type = type;
+ inner.children = (kmp_taskgraph_region **)(this + 1); // trailing storage
+ inner.num_children = num_nodes;
+ this->parent = parent;
+ *alloc_chain = this;
+ alloc_chain = &this->alloc_chain;
+}
+
+[[maybe_unused]] static kmp_taskgraph_region_t *__kmp_taskgraph_region_alloc(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_node_t *node,
+ kmp_taskgraph_region_t *parent) {
+ return new (thread)
+ kmp_taskgraph_region_t(taskgraph, alloc_chain, node, parent);
+}
+
+static kmp_taskgraph_region_t *__kmp_taskgraph_region_alloc(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t **&alloc_chain, enum kmp_taskgraph_region_type type,
+ kmp_int32 num_nodes, kmp_taskgraph_region_t *parent) {
+ return new (thread, num_nodes)
+ kmp_taskgraph_region_t(taskgraph, alloc_chain, type, num_nodes, parent);
+}
+
+kmp_int32 __kmp_taskgraph_topological_order(kmp_taskgraph_region_t *region,
+ kmp_taskgraph_region_t **order_out,
+ kmp_int32 *outidx) {
+ if (region->mark == TASKGRAPH_PERMANENT_MARK)
+ return region->level;
+
+ assert(region->mark != TASKGRAPH_TEMP_MARK);
+
+ region->mark = TASKGRAPH_TEMP_MARK;
+
+ kmp_int32 max_level = -1;
+ for (kmp_taskgraph_region_dep_t *s = region->predecessors; s; s = s->next) {
+ kmp_int32 pred_level =
+ __kmp_taskgraph_topological_order(s->region, order_out, outidx);
+ max_level = pred_level > max_level ? pred_level : max_level;
+ }
+
+ region->level = max_level + 1;
+ region->mark = TASKGRAPH_PERMANENT_MARK;
+ order_out[(*outidx)++] = region;
+
+ return region->level;
+}
+
+static void
+__kmp_taskgraph_region_chain_clear_marks(kmp_taskgraph_region_t *region) {
+ for (; region; region = region->next)
+ region->mark = TASKGRAPH_UNMARKED;
+}
+
+static void
+__kmp_taskgraph_region_chain_prune(kmp_taskgraph_region_t **region_p) {
+ kmp_taskgraph_region_t *pruned_region = nullptr, *region = *region_p;
+ kmp_taskgraph_region_t **pruned_region_p = &pruned_region;
+
+ TGDBG("pruning worklist...\n");
+
+ // NOTE: Pruning and deletion look the same here with respect to the handling
+ // of the worklist, but deleted nodes are freed from the taskgraph structure
+ // during cleanup, whereas combined nodes are retained.
+ for (; region; region = region->next) {
+ if (region->mark == TASKGRAPH_COMBINED || region->mark == TASKGRAPH_DELETED)
+ *pruned_region_p = region->next;
+ else {
+ *pruned_region_p = region;
+ pruned_region_p = ®ion->next;
+ }
+ }
+
+ *pruned_region_p = nullptr;
+ *region_p = pruned_region;
+}
+
+static kmp_int32 __kmp_region_deplist_len(kmp_taskgraph_region_dep_t *list) {
+ kmp_int32 len = 0;
+ for (; list; list = list->next)
+ ++len;
+ return len;
+}
+
+void __kmp_region_deplist_free(kmp_info_t *thread,
+ kmp_taskgraph_region_dep_t *list) {
+ while (list) {
+ kmp_taskgraph_region_dep_t *next = list->next;
+ __kmp_fast_free(thread, list);
+ list = next;
+ }
+}
+
+static void __kmp_region_dep_recycle(kmp_taskgraph_region_dep_t **recycled,
+ kmp_taskgraph_region_dep_t *dep) {
+ dep->next = *recycled;
+ *recycled = dep;
+}
+
+void __kmp_region_deplist_recycle(kmp_taskgraph_region_dep_t **recycled,
+ kmp_taskgraph_region_dep_t *list) {
+ while (list) {
+ kmp_taskgraph_region_dep_t *next = list->next;
+ __kmp_region_dep_recycle(recycled, list);
+ list = next;
+ }
+}
+
+// Release the dependency lists of every region in the finished region tree.
+//
+// The direct children of an IRREDUCIBLE region keep their lists. We release
+// those when we free the recorded taskgraph (in
+// __kmp_taskgraph_free_region_metadata).
+static void
+__kmp_taskgraph_free_built_region_deps(kmp_info_t *thread,
+ kmp_taskgraph_region_t *region) {
+ if (!region->parent || region->parent->type != TASKGRAPH_REGION_IRREDUCIBLE) {
+ __kmp_region_deplist_free(thread, region->predecessors);
+ __kmp_region_deplist_free(thread, region->successors);
+ region->predecessors = nullptr;
+ region->successors = nullptr;
+ }
+ switch (region->type) {
+ case TASKGRAPH_REGION_PARALLEL:
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ case TASKGRAPH_REGION_IRREDUCIBLE:
+ for (kmp_int32 k = 0; k < region->inner.num_children; k++)
+ __kmp_taskgraph_free_built_region_deps(thread, region->inner.children[k]);
+ break;
+ default:
+ break;
+ }
+}
+
+static bool __kmp_taskgraph_collapse_sequence(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t **region_p,
+ kmp_taskgraph_region_t *parent, kmp_int32 &stamp) {
+ kmp_taskgraph_region_t *region = *region_p;
+ kmp_taskgraph_region_t *chain_start = region;
+ kmp_taskgraph_region_t *chain_end = region;
+ kmp_int32 chain_len = 1;
+
+ if (region->type == TASKGRAPH_REGION_ENTRY)
+ return false;
+
+ while (__kmp_region_deplist_len(chain_end->successors) == 1) {
+ kmp_taskgraph_region_t *past_end = chain_end->successors->region;
+ if (__kmp_region_deplist_len(past_end->predecessors) == 1) {
+ if (past_end->type == TASKGRAPH_REGION_EXIT)
+ break;
+ else {
+ chain_end = past_end;
+ ++chain_len;
+ }
+ } else
break;
+ }
+
+ if (chain_len <= 1)
+ return false;
+
+ auto *seq_region = __kmp_taskgraph_region_alloc(
+ thread, taskgraph, alloc_chain, TASKGRAPH_REGION_SEQUENTIAL, chain_len,
+ parent);
+ TGDBG("allocated new seq region: %p (length %d)\n", seq_region, chain_len);
+ kmp_taskgraph_region_t **worklist_p = region_p;
+ *worklist_p = seq_region;
+ seq_region->next = chain_start->next;
+ kmp_int32 level = -1;
+ for (kmp_int32 i = 0; i < chain_len; i++) {
+ seq_region->inner.children[i] = chain_start;
+ TGDBG("mark node %p as combined\n", chain_start);
+ chain_start->mark = TASKGRAPH_COMBINED;
+ chain_start->timestamp = stamp;
+ chain_start->parent = seq_region;
+ // The level of the sequence is the level of the first node.
+ if (level == -1)
+ level = chain_start->level;
+
+ if (i < chain_len - 1) {
+ chain_start = chain_start->successors->region;
+ }
+ }
+
+ seq_region->level = level;
+ seq_region->predecessors = seq_region->inner.children[0]->predecessors;
+ seq_region->successors =
+ seq_region->inner.children[chain_len - 1]->successors;
+ seq_region->inner.children[0]->predecessors = nullptr;
+ seq_region->inner.children[chain_len - 1]->successors = nullptr;
+
+ // Update predecessors to point to new seq region.
+ for (kmp_taskgraph_region_dep_t *pred = seq_region->predecessors; pred;
+ pred = pred->next) {
+ for (kmp_taskgraph_region_dep_t *succ = pred->region->successors; succ;
+ succ = succ->next) {
+ if (succ->region == seq_region->inner.children[0]) {
+ succ->region = seq_region;
+ }
+ }
+ }
+
+ // Update successors to point back to new seq region.
+ for (kmp_taskgraph_region_dep_t *succ = seq_region->successors; succ;
+ succ = succ->next) {
+ for (kmp_taskgraph_region_dep_t *pred = succ->region->predecessors; pred;
+ pred = pred->next) {
+ if (pred->region == seq_region->inner.children[chain_len - 1]) {
+ pred->region = seq_region;
+ }
+ }
+ }
+
+ return true;
+}
+
+static const char *
+__kmp_taskgraph_region_type_name(kmp_taskgraph_region_type type);
+
+static void __kmp_taskgraph_region_dfs(kmp_taskgraph_region_t *region,
+ kmp_taskgraph_region_t **order,
+ kmp_int32 &idx, bool use_preds) {
+ if (order) {
+ region->timestamp = --idx;
+ order[idx] = region;
+ }
+ region->mark = TASKGRAPH_TEMP_MARK;
+ for (kmp_taskgraph_region_dep_t *reg = use_preds ? region->predecessors
+ : region->successors;
+ reg; reg = reg->next) {
+ if (reg->region->mark == TASKGRAPH_UNMARKED)
+ __kmp_taskgraph_region_dfs(reg->region, order, idx, use_preds);
+ }
+}
+
+#if defined(DEBUG_TASKGRAPH) && defined(CHECK_WORKLIST)
+
+static void __kmp_taskgraph_region_gather_deps(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t *region, kmp_taskgraph_region_dep_t **deplist,
+ bool &ok) {
+ for (kmp_taskgraph_region_dep_t *dep = *deplist; dep; dep = dep->next) {
+ if (dep->region == region)
+ return;
+ }
+
+ *deplist = __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, region,
+ *deplist);
+
+ for (kmp_taskgraph_region_dep_t *pred = region->predecessors; pred;
+ pred = pred->next) {
+ if (pred->region->mark == TASKGRAPH_DELETED) {
+ fprintf(stderr, "*** Region %p's predecessor %p is a deleted node\n",
+ region, pred->region);
+ ok = false;
+ }
+ __kmp_taskgraph_region_gather_deps(thread, taskgraph, pred->region, deplist,
+ ok);
+ }
+
+ for (kmp_taskgraph_region_dep_t *succ = region->successors; succ;
+ succ = succ->next) {
+ if (succ->region->mark == TASKGRAPH_DELETED) {
+ fprintf(stderr, "*** Region %p's successor %p is a deleted node\n",
+ region, succ->region);
+ ok = false;
+ }
+ __kmp_taskgraph_region_gather_deps(thread, taskgraph, succ->region, deplist,
+ ok);
+ }
+}
+
+static bool __kmp_taskgraph_region_worklist_check(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t *region, const char *where) {
+ kmp_taskgraph_region_dep_t *collected_nodes = nullptr;
+ bool ok = true;
+ __kmp_taskgraph_region_gather_deps(thread, taskgraph, region,
+ &collected_nodes, ok);
+
+ // Check all collected nodes are in the region's worklist.
+ for (kmp_taskgraph_region_dep_t *cn = collected_nodes; cn; cn = cn->next) {
+ bool in_list = false;
+ for (kmp_taskgraph_region_t *r = region; r; r = r->next) {
+ if (r == cn->region) {
+ in_list = true;
+ break;
+ }
+ }
+ if (!in_list) {
+ fprintf(stderr,
+ "*** Region %p is in dependency graph but not worklist (%s)\n",
+ cn->region, where);
+ ok = false;
+ }
+ }
+
+ for (kmp_taskgraph_region_t *r = region; r; r = r->next) {
+ bool in_list = false;
+ for (kmp_taskgraph_region_dep_t *cn = collected_nodes; cn; cn = cn->next) {
+ if (r == cn->region) {
+ in_list = true;
+ break;
+ }
+ }
+ if (!in_list) {
+ fprintf(stderr,
+ "*** Region %p is in worklist but not dependency graph (%s)\n", r,
+ where);
+ ok = false;
+ }
+ }
+
+ __kmp_region_deplist_recycle(&taskgraph->recycled_deps, collected_nodes);
+
+ return ok;
+}
+#else
+static bool __kmp_taskgraph_region_worklist_check(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t *region, const char *where) {
+ return true;
+}
+#endif
+
+static kmp_taskgraph_region_t *__kmp_taskgraph_region_dom_intersect(
+ kmp_taskgraph_region_t **order, kmp_taskgraph_region_t **doms,
+ kmp_taskgraph_region_t *b1, kmp_taskgraph_region_t *b2) {
+ kmp_int32 finger1 = b1->timestamp;
+ kmp_int32 finger2 = b2->timestamp;
+ while (finger1 != finger2) {
+ while (finger1 < finger2)
+ finger1 = doms[finger1]->timestamp;
+ while (finger2 < finger1)
+ finger2 = doms[finger2]->timestamp;
+ }
+ return order[finger1];
+}
+
+static void __kmp_taskgraph_region_doms(kmp_taskgraph_region_t **order,
+ kmp_taskgraph_region_t **doms,
+ kmp_int32 worklist_length,
+ bool postdom) {
+ bool changed = true;
+ // Set doms[start_node] <- start_node
+ doms[worklist_length - 1] = order[worklist_length - 1];
+ order[worklist_length - 1]->mark = TASKGRAPH_PERMANENT_MARK;
+ while (changed) {
+ changed = false;
+ for (int n = 0; n < worklist_length - 1; n++) {
+ kmp_taskgraph_region_t *b = order[n];
+ kmp_taskgraph_region_t *new_idom = nullptr;
+ for (kmp_taskgraph_region_dep_t *pred = postdom ? b->successors
+ : b->predecessors;
+ pred; pred = pred->next) {
+ if (pred->region->mark == TASKGRAPH_PERMANENT_MARK) {
+ new_idom = pred->region;
+ break;
+ }
+ }
+ for (kmp_taskgraph_region_dep_t *pred = postdom ? b->successors
+ : b->predecessors;
+ pred; pred = pred->next) {
+ if (pred->region == new_idom)
+ continue;
+ if (doms[pred->region->timestamp]) {
+ new_idom = __kmp_taskgraph_region_dom_intersect(
+ order, doms, pred->region, new_idom);
+ }
+ }
+ if (doms[b->timestamp] != new_idom) {
+ doms[b->timestamp] = new_idom;
+ order[b->timestamp]->mark = TASKGRAPH_PERMANENT_MARK;
+ changed = true;
+ }
+ }
+ }
+}
+
+static bool __kmp_taskgraph_region_mutex_p(kmp_taskgraph_region_t *reg) {
+ if (reg->type == TASKGRAPH_REGION_NODE)
+ return reg->mutexset != nullptr;
+ return false;
+}
+
+// This function collapses graph regions with forms like this:
+//
+// 1. A(pp) 2. A 3. A(pp)
+// / \ / \ / \
+// B C B(pp) E(pp) B(pp) E
+// \ / / \ / \ / \ /
+// D(*) C D F G C D /
+// \ \ / / \ | /
+// `---H(*)--' F(*)
+//
+// We look for a node with more than one predecessor (*), where each of those
+// predecessors has a single successor and a single predecessor (pp). We group
+// nodes by which pp (predecessor-predecessor) they have: for (1), nodes B & C
+// share a pp; for (2), C & D share a pp, and F & G share a pp; for (3), C & D
+// share a pp, and E has a separate pp.
+//
+// We choose the pp with the highest level ("furthest down the graph"), and
+// collapse the subgraph into a parallel region.
+
+static bool __kmp_taskgraph_collapse_par_exclusive(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t **region_p,
+ kmp_taskgraph_region_t *parent, kmp_int32 &stamp) {
+ kmp_taskgraph_region_t *region = *region_p;
+ kmp_int32 num_predecessors = __kmp_region_deplist_len(region->predecessors);
+
+ TGDBG("predecessors %d, successors %d\n",
+ __kmp_region_deplist_len(region->predecessors),
+ __kmp_region_deplist_len(region->successors));
+
+ if (num_predecessors <= 1)
+ return false;
+
+ TGDBG("found multiple predecessors, creating parallel/unordered region\n");
+ kmp_taskgraph_region_dep_t *pred_preds = nullptr;
+ kmp_int32 highest_level = -1;
+
+ for (kmp_taskgraph_region_dep_t *pred = region->predecessors; pred;
+ pred = pred->next) {
+ TGDBG("consider predecessor: %p\n", pred->region);
+ TGDBG("-- successors %d, predecessors %d\n",
+ __kmp_region_deplist_len(pred->region->successors),
+ __kmp_region_deplist_len(pred->region->predecessors));
+ if (highest_level == -1 || pred->region->level > highest_level)
+ highest_level = pred->region->level;
+ kmp_taskgraph_region_t *pred_region = pred->region;
+ if (__kmp_region_deplist_len(pred_region->successors) != 1)
+ continue;
+ if (__kmp_region_deplist_len(pred_region->predecessors) != 1)
+ continue;
+ bool in_list = false;
+ TGDBG("pp region: %p (%s)\n", pred_region->predecessors->region,
+ __kmp_taskgraph_region_type_name(
+ pred_region->predecessors->region->type));
+ kmp_taskgraph_region_t *pp_region = pred_region->predecessors->region;
+ for (kmp_taskgraph_region_dep_t *pp = pred_preds; pp; pp = pp->next) {
+ if (pp->region == pp_region) {
+ in_list = true;
+ break;
+ }
+ }
+ if (!in_list) {
+ pred_preds = __kmp_region_deplist_add(thread, &taskgraph->recycled_deps,
+ pp_region, pred_preds);
+ TGDBG("add %p to list: len(pred_preds)=%d\n", pp_region,
+ __kmp_region_deplist_len(pred_preds));
}
}
+ kmp_int32 num_pps = __kmp_region_deplist_len(pred_preds);
+ if (num_pps == 0) {
+ TGDBG("no collapsible regions, bailing out\n");
+ return false;
+ }
+ TGDBG("found %d predecessor-predecessors\n", num_pps);
+ TGDBG("highest pred level: %d\n", highest_level);
+
+ bool changed = false;
+
+ for (kmp_taskgraph_region_dep_t *pp = pred_preds; pp; pp = pp->next) {
+ kmp_taskgraph_region_dep_t *par_succs = nullptr;
+ kmp_taskgraph_region_dep_t *par_preds = nullptr;
+ kmp_int32 preds_for_pp = 0;
+ bool any_mutex_p = false;
+ for (kmp_taskgraph_region_dep_t *pred = region->predecessors; pred;
+ pred = pred->next) {
+ kmp_taskgraph_region_t *pred_region = pred->region;
+ if (!pred_region->predecessors)
+ continue;
+ if (pred_region->level < highest_level)
+ continue;
+ if (__kmp_region_deplist_len(pred_region->predecessors) != 1 ||
+ __kmp_region_deplist_len(pred_region->successors) != 1)
+ continue;
+ TGDBG("counting pred region: %p (%s)\n", pred_region,
+ __kmp_taskgraph_region_type_name(pred_region->type));
+ if (pred_region->predecessors->region == pp->region) {
+ ++preds_for_pp;
+ if (__kmp_taskgraph_region_mutex_p(pred_region))
+ any_mutex_p = true;
+ }
+ }
+ TGDBG("found %d preds for pp region %p\n", preds_for_pp, pp->region);
+ if (preds_for_pp < 2)
+ continue;
+ kmp_taskgraph_region_type region_type =
+ any_mutex_p ? TASKGRAPH_REGION_EXCLUSIVE : TASKGRAPH_REGION_PARALLEL;
+ auto *par_region = __kmp_taskgraph_region_alloc(
+ thread, taskgraph, alloc_chain, region_type, preds_for_pp, parent);
+ changed = true;
+ TGDBG("allocated %s region: %p\n",
+ region_type == TASKGRAPH_REGION_EXCLUSIVE ? "exclusive" : "parallel",
+ par_region);
+ kmp_taskgraph_region_dep_t *pred = region->predecessors;
+ kmp_int32 level = -1;
+ bool found_reduction_data = false;
+ for (kmp_int32 i = 0; pred; pred = pred->next) {
+ kmp_taskgraph_region_t *pred_region = pred->region;
+ TGDBG("considering pred region: %p\n", pred_region);
+ if (!pred_region->predecessors) {
+ TGDBG("bailing (no predecessors)\n");
+ continue;
+ }
+ if (pred_region->predecessors->region != pp->region) {
+ TGDBG("bailing (wrong pp region)\n");
+ continue;
+ }
+ if (__kmp_region_deplist_len(pred_region->predecessors) != 1 ||
+ __kmp_region_deplist_len(pred_region->successors) != 1) {
+ TGDBG("bailing (non-unit pred/succ list length)\n");
+ continue;
+ }
+ TGDBG("process region %p (%d/%d), level %d\n", pred->region, i + 1,
+ preds_for_pp, pred_region->level);
+ par_region->inner.children[i] = pred_region;
+ pred_region->mark = TASKGRAPH_COMBINED;
+ pred_region->timestamp = stamp;
+ pred_region->parent = par_region;
+
+ // Reduction handling. The reduction input data is now attached to one
+ // of the tasks participating in the reduction. Move it to the enclosing
+ // parallel region instead.
+ if (pred_region->type == TASKGRAPH_REGION_NODE &&
+ pred_region->task.node->reduce_input) {
+ // We should only be doing this once per par region.
+ assert(!par_region->reduce_input);
+ par_region->reduce_input = pred_region->task.node->reduce_input;
+ pred_region->task.node->reduce_input = nullptr;
+ found_reduction_data = true;
+ }
+
+ // We expect all the predecessor regions to be at the same level.
+ if (level == -1)
+ level = pred_region->level;
+ else
+ assert(level == pred_region->level);
+ if (!par_succs) {
+ // Copy one list of predecessors/successors for the predecessor region.
+ // We know these are of length one by checks above. We'll re-use them
+ // for the created parallel region.
+ par_preds = pred_region->predecessors;
+ par_succs = pred_region->successors;
+ pred_region->predecessors = nullptr;
+ pred_region->successors = nullptr;
+ }
+ i++;
+ }
+ par_region->level = level;
+ par_region->predecessors = par_preds;
+ par_region->successors = par_succs;
+
+ if (region->type == TASKGRAPH_REGION_WAIT && !found_reduction_data) {
+ // If we have no reduction data, we will not create a taskgroup for this
+ // parallel region at replay time, so we don't need to terminate/discard
+ // that region when we're done. Clear the taskloop_task flag.
+ region->task.node->taskloop_task = false;
+ }
+
+ // Add the new parallel region to the worklist. FIXME: We're reprocessing
+ // the 'region' node here -- we don't need to do that if it's fully
+ // consumed.)
+ par_region->next = region->next;
+ region->next = par_region;
+ }
+
+#ifdef DEBUG_TASKGRAPH
+ TGDBG("before pred fixup:\n");
+ for (kmp_taskgraph_region_dep_t *pred = region->predecessors; pred;
+ pred = pred->next) {
+ TGDBG("region %p, pred region: %p\n", region, pred->region);
+ }
+#endif
+
+ // Now, fix up predecessor list for 'region', and successor lists for each
+ // predecessor-predecessor.
+ kmp_taskgraph_region_dep_t **dep_p = ®ion->predecessors;
+ while (*dep_p) {
+ kmp_taskgraph_region_dep_t *dep = *dep_p;
+ if (dep->region->mark == TASKGRAPH_COMBINED) {
+ if (!dep->region->successors) {
+ dep->region = dep->region->parent;
+ dep_p = &dep->next;
+ } else {
+ kmp_taskgraph_region_dep_t *next = dep->next;
+ __kmp_region_dep_recycle(&taskgraph->recycled_deps, dep);
+ *dep_p = next;
+ }
+ } else {
+ dep_p = &dep->next;
+ }
+ }
+
+#ifdef DEBUG_TASKGRAPH
+ TGDBG("after pred fixup:\n");
+ for (kmp_taskgraph_region_dep_t *pred = region->predecessors; pred;
+ pred = pred->next) {
+ TGDBG("region %p, pred region: %p\n", region, pred->region);
+ }
+#endif
+
+ for (kmp_taskgraph_region_dep_t *pp = pred_preds; pp; pp = pp->next) {
+ kmp_taskgraph_region_t *pp_region = pp->region;
+ dep_p = &pp_region->successors;
+ while (*dep_p) {
+ kmp_taskgraph_region_dep_t *dep = *dep_p;
+ if (dep->region->mark == TASKGRAPH_COMBINED) {
+ if (!dep->region->predecessors) {
+ dep->region = dep->region->parent;
+ dep_p = &dep->next;
+ } else {
+ kmp_taskgraph_region_dep_t *next = dep->next;
+ __kmp_region_dep_recycle(&taskgraph->recycled_deps, dep);
+ *dep_p = next;
+ }
+ } else {
+ dep_p = &dep->next;
+ }
+ }
+ }
+
+ __kmp_region_deplist_recycle(&taskgraph->recycled_deps, pred_preds);
+
+ return changed;
+}
+
+[[maybe_unused]] static void
+__kmp_taskgraph_region_dot(kmp_taskgraph_region_t *region, const char *name) {
+ fprintf(stderr, "digraph %s {\n", name);
+ for (kmp_taskgraph_region_t *r = region; r; r = r->next) {
+ if (r->mark == TASKGRAPH_DELETED) {
+ fprintf(stderr, "\"%p\" [shape=box, label=\"%p(%s) (deleted)\"]\n", r, r,
+ __kmp_taskgraph_region_type_name(r->type));
+ } else if (r->level == -1) {
+ fprintf(stderr, "\"%p\" [shape=box, label=\"%p(%s) (new)\"]\n", r, r,
+ __kmp_taskgraph_region_type_name(r->type));
+ } else {
+ fprintf(stderr, "\"%p\" [shape=box, label=\"%p(%s)\"]\n", r, r,
+ __kmp_taskgraph_region_type_name(r->type));
+ }
+ for (kmp_taskgraph_region_dep_t *succ = r->successors; succ;
+ succ = succ->next) {
+ fprintf(stderr, " \"%p\" -> \"%p\" [color=green]\n", r, succ->region);
+ }
+ for (kmp_taskgraph_region_dep_t *pred = r->predecessors; pred;
+ pred = pred->next) {
+ fprintf(stderr, " \"%p\" -> \"%p\" [color=red, constraint=false]\n", r,
+ pred->region);
+ }
+ }
+ fprintf(stderr, "}\n");
+}
+
+/// Wrap a single-entry / single-exit (SESE) irreducible region into one
+/// TASKGRAPH_REGION_IRREDUCIBLE region.
+//
+// `entryregion` and `exitregion` are the SESE region's boundary
+// sentinels, and `members` (length `num_children`) are its interior nodes,
+// which we know at this point to be non-series/parallel reducible. Because
+// the region is SESE, every edge into the interior comes through the entry and
+// every edge out of it leaves through the exit, so we can collapse the
+// interior into one node without disturbing anything outside the region.
+//
+// We collect the members as the children of one IRREDUCIBLE region and
+// *retain* their child<->child (intra-kernel) predecessor/successor edges --
+// those edges are the kernel's internal DAG. Edges to the entry/exit
+// sentinels are dropped from the children and replaced by a single
+// entry -> irreducible -> exit bracket, so the surrounding graph becomes
+// trivially series-parallel around the kernel and the build loop makes
+// progress.
+
+static kmp_taskgraph_region_t *__kmp_taskgraph_carve_irreducible(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t *entryregion,
+ kmp_taskgraph_region_t *exitregion, kmp_taskgraph_region_t **members,
+ kmp_int32 num_children) {
+ auto *irr = __kmp_taskgraph_region_alloc(thread, taskgraph, alloc_chain,
+ TASKGRAPH_REGION_IRREDUCIBLE,
+ num_children, nullptr);
+
+ for (kmp_int32 c = 0; c < num_children; c++) {
+ kmp_taskgraph_region_t *r = members[c];
+ irr->inner.children[c] = r;
+ r->mark = TASKGRAPH_COMBINED;
+ r->parent = irr;
+ // We've clobbered level info already: this isn't valid now, so don't
+ // pretend it is.
+ r->level = -1;
+
+ // Drop the child's edges to the entry/exit sentinels; intra-kernel edges
+ // (to other children) are kept as the irreducible region's structure.
+ kmp_taskgraph_region_dep_t **predp = &r->predecessors;
+ while (*predp) {
+ if ((*predp)->region == entryregion) {
+ kmp_taskgraph_region_dep_t *dead = *predp;
+ *predp = dead->next;
+ __kmp_region_dep_recycle(&taskgraph->recycled_deps, dead);
+ } else {
+ predp = &(*predp)->next;
+ }
+ }
+ kmp_taskgraph_region_dep_t **succp = &r->successors;
+ while (*succp) {
+ if ((*succp)->region == exitregion) {
+ kmp_taskgraph_region_dep_t *dead = *succp;
+ *succp = dead->next;
+ __kmp_region_dep_recycle(&taskgraph->recycled_deps, dead);
+ } else {
+ succp = &(*succp)->next;
+ }
+ }
+ }
+ irr->level = -1;
+
+ // Bracket the kernel: entry -> irr -> exit (collapse the fan-in/out into a
+ // single edge each way). Because the region is SESE, every successor of the
+ // entry leads into the interior and every predecessor of the exit comes from
+ // the interior, so replacing those lists wholesale is safe.
+ __kmp_region_deplist_recycle(&taskgraph->recycled_deps,
+ entryregion->successors);
+ entryregion->successors =
+ __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, irr, nullptr);
+ __kmp_region_deplist_recycle(&taskgraph->recycled_deps,
+ exitregion->predecessors);
+ exitregion->predecessors =
+ __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, irr, nullptr);
+ irr->predecessors = __kmp_region_deplist_add(
+ thread, &taskgraph->recycled_deps, entryregion, nullptr);
+ irr->successors = __kmp_region_deplist_add(thread, &taskgraph->recycled_deps,
+ exitregion, nullptr);
+
+ // Splice the irreducible region into the worklist right after the entry
+ // sentinel; the now-COMBINED children are pruned by the caller's next pass.
+ irr->next = entryregion->next;
+ entryregion->next = irr;
+ return irr;
+}
+
+// Decompose the irreducible residual into its nested single-entry / single-exit
+// (SESE) regions using immediate dominators and postdominators, and carve each
+// minimal (leaf) SESE region into its own IRREDUCIBLE node. Only the genuine
+// non-series-parallel knots become IRREDUCIBLE; the surrounding reducible /
+// reduction structure is left for the series/parallel collapse to handle on a
+// subsequent pass of the build loop. Returns true if at least one region was
+// carved.
+//
+// On a DAG with a single source (ENTRY) and single sink (EXIT), a pair
+// (entry, exit) bounds a canonical SESE region iff exit = ipdom[entry] and
+// entry = idom[exit]. Read: every path leaving the entry reconverges at the
+// (single) exit and every path reaching the exit comes through the (single)
+// entry, so everything strictly between the entry and exit is enterable
+// only via the entry and leavable only via the exit. These pairs form a
+// laminar (properly nested) family; a leaf is one whose interior contains no
+// other pair's entry.
+//
+// After the series/parallel collapse and the transitive-drop / twin-merge
+// passes have run to fixpoint, any surviving leaf region is genuinely
+// irreducible -- a clean series/parallel interior would already have been
+// collapsed -- so carving leaves is exactly right and preserves everything
+// reducible around the knots.
+
+static bool __kmp_taskgraph_carve_sese(kmp_info_t *thread,
+ kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t **&alloc_chain,
+ kmp_taskgraph_region_t **order,
+ kmp_taskgraph_region_t **idom,
+ kmp_int32 worklist_length,
+ kmp_taskgraph_region_t *exitregion) {
+ kmp_int32 n = worklist_length;
+
+ // At the start, we already have immediate-dominator information. The
+ // 'order' array points to regions in reverse postorder, and each region
+ // has a timestamp field that contains its own index in that array.
+ // The 'idom' array is indexed by reverse-postorder (i.e. the region's
+ // timestamp field), which means we should copy that timestamp field before
+ // the second (post) dominator-finding calls below. We're not otherwise
+ // using the level field at this point.
+ for (kmp_int32 i = 0; i < n; i++) {
+ order[i]->level = order[i]->timestamp;
+ }
+
+ // Compute immediate postdominators. Mirror the dominator setup: DFS from the
+ // EXIT sentinel following predecessors so the exit lands at index n-1, then
+ // run the same dominator solver in postdom mode (which walks successors).
+ // Reset marks first: after the dominator solve every region is PERMANENT, but
+ // region_dfs only recurses into UNMARKED neighbours.
+ kmp_taskgraph_region_t **order_pd =
+ (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+ thread, n * sizeof(kmp_taskgraph_region_t *));
+ kmp_taskgraph_region_t **ipdom =
+ (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+ thread, n * sizeof(kmp_taskgraph_region_t *));
+ memset(ipdom, 0, n * sizeof(kmp_taskgraph_region_t *));
+ for (kmp_int32 i = 0; i < n; i++)
+ order[i]->mark = TASKGRAPH_UNMARKED;
+ kmp_int32 cursor = n;
+ __kmp_taskgraph_region_dfs(exitregion, order_pd, cursor, /*use_preds=*/true);
+ assert(cursor == 0);
+ __kmp_taskgraph_region_doms(order_pd, ipdom, n, /*postdom=*/true);
+
+#ifdef DEBUG_TASKGRAPH
+ fprintf(stderr, "digraph {\n");
+ for (kmp_int32 i = 0; i < worklist_length; i++) {
+ kmp_taskgraph_region_t *b = order[i];
+ for (kmp_taskgraph_region_dep_t *succ = b->successors; succ;
+ succ = succ->next) {
+ fprintf(stderr, " \"%d\" -> \"%d\"\n", b->timestamp,
+ succ->region->timestamp);
+ }
+ fprintf(stderr, " \"%d\" -> \"%d\" [color=green, constraint=false]\n",
+ b->timestamp, idom[b->timestamp]->timestamp);
+ fprintf(stderr, " \"%d\" -> \"%d\" [color=red, constraint=false]\n",
+ b->timestamp, ipdom[b->timestamp]->timestamp);
+ }
+ fprintf(stderr, "}\n");
+#endif
+
+ auto dominates = [&](kmp_taskgraph_region_t *a,
+ kmp_taskgraph_region_t *b) -> bool {
+ kmp_taskgraph_region_t *cur = b;
+ while (true) {
+ // A node dominates itself.
+ if (cur == a)
+ return true;
+ kmp_taskgraph_region_t *anc = idom[cur->level];
+ // We've reached the entry node without encountering 'a':
+ // 'a' does not dominate 'b'.
+ if (anc == cur)
+ return false;
+ cur = anc;
+ }
+ };
+ auto postdominates = [&](kmp_taskgraph_region_t *a,
+ kmp_taskgraph_region_t *b) -> bool {
+ kmp_taskgraph_region_t *cur = b;
+ while (true) {
+ // A node postdominates itself.
+ if (cur == a)
+ return true;
+ kmp_taskgraph_region_t *anc = ipdom[cur->timestamp];
+ // We've reached the exit node without encountering 'a':
+ // 'a' does not postdominate 'b'.
+ if (anc == cur)
+ return false;
+ cur = anc;
+ }
+ };
+
+ // Matched SESE pairs: matched[i] is pdom(i) if matched[i] starts the SESE
+ // region and pdom(i) ends it.
+ kmp_taskgraph_region_t **matched =
+ (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+ thread, n * sizeof(kmp_taskgraph_region_t *));
+ for (kmp_int32 i = 0; i < n; i++) {
+ matched[i] = nullptr;
+ kmp_taskgraph_region_t *region = order[i];
+ kmp_taskgraph_region_t *pdom = ipdom[region->timestamp];
+ // Only true for exit region.
+ if (region == pdom)
+ continue;
+ if (idom[pdom->level] == region)
+ matched[i] = pdom;
+ }
+
+ // Carve every leaf pair with a non-empty interior. Leaves are vertex-
+ // disjoint and each carve only rewires edges incident to its own interior,
+ // so carving them all in one pass is safe. Membership uses the pre-carve
+ // idom/ipdom relations, which the carve does not modify.
+ bool carved = false;
+ kmp_taskgraph_region_t **members =
+ (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+ thread, n * sizeof(kmp_taskgraph_region_t *));
+ for (kmp_int32 i = 0; i < n; i++) {
+ if (!matched[i])
+ continue;
+ kmp_taskgraph_region_t *entry = order[i];
+ kmp_taskgraph_region_t *exit = matched[i];
+
+ // Leaf test: no other matched pair's entry lies strictly inside
+ // (entry, exit).
+ bool is_leaf = true;
+ for (kmp_int32 k = 0; k < n && is_leaf; k++) {
+ if (k == i || !matched[k])
+ continue;
+ kmp_taskgraph_region_t *region = order[k];
+ if (region == entry || region == exit)
+ continue;
+ if (dominates(entry, region) && postdominates(exit, region))
+ is_leaf = false;
+ }
+ if (!is_leaf)
+ continue;
+
+ // Gather the interior: nodes dominated by entry and postdominated by exit,
+ // other than the boundary sentinels (and skipping anything already carved).
+ kmp_int32 num_members = 0;
+ for (kmp_int32 m = 0; m < n; m++) {
+ kmp_taskgraph_region_t *region = order[m];
+ if (region == entry || region == exit ||
+ region->mark == TASKGRAPH_COMBINED)
+ continue;
+ if (dominates(entry, region) && postdominates(exit, region)) {
+ TGDBG("carving member: %d\n", region->timestamp);
+ members[num_members++] = region;
+ }
+ }
+ if (num_members == 0)
+ continue;
+
+ TGDBG("carving %d nodes from %d to %d\n", num_members, entry->timestamp,
+ exit->timestamp);
+ __kmp_taskgraph_carve_irreducible(thread, taskgraph, alloc_chain, entry,
+ exit, members, num_members);
+ carved = true;
+ }
+
+ __kmp_fast_free(thread, members);
+ __kmp_fast_free(thread, matched);
+ __kmp_fast_free(thread, ipdom);
+ __kmp_fast_free(thread, order_pd);
+ return carved;
+}
+
+/// This function uses several strategies to turn an irreducible taskgraph
+/// into a reducible taskgraph.
+//
+// 1. If a node C depends on node B and also node A which dominates C,
+// and if B also dominates C, then the dependency of C on A can be dropped.
+// That is, we know B must execute after A, so we can say execution must
+// proceed A->B->C, and we don't also need to specify the transitive A->C
+// dependency directly.
+//
+// A A
+// / \ |
+// B ) -> B
+// \ / |
+// C C
+//
+// 2. Two nodes with the same set of predecessors and successors are turned
+// into a parallel region. This graph form can arise from use of
+// "inoutset" dependencies.
+//
+// A B C A B C
+// / \/ \/ \ | | |
+// /__/\ /\__\ | | |
+// D____X____E (A+B+C->D & A+B+C->E) -> par(D,E)
+// \ / |
+// '--F--' F
+//
+// 3. Otherwise the residual is genuinely non-series-parallel (no series,
+// parallel, transitive-drop or twin merge applies). We decompose the
+// residual into its nested single-entry / single-exit (SESE) regions via
+// immediate dominators/postdominators (__kmp_taskgraph_carve_sese) and
+// carve only the minimal (leaf) irreducible regions into
+// TASKGRAPH_REGION_IRREDUCIBLE nodes. See the commentary on
+// __kmp_taskgraph_carve_sese and __kmp_taskgraph_carve_irreducible.
+
+static bool __kmp_taskgraph_rewrite_irreducible(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t **region_p,
+ kmp_taskgraph_region_t *exitregion) {
+ kmp_taskgraph_region_t *entryregion = *region_p;
+
+ kmp_int32 worklist_length = 0;
+ for (kmp_taskgraph_region_t *r = entryregion; r; r = r->next) {
+ // Deleted regions stay deleted. (We could actually remove these from
+ // the worklist here, I think.)
+ if (r->mark == TASKGRAPH_DELETED)
+ continue;
+ r->mark = TASKGRAPH_UNMARKED;
+ worklist_length++;
+ }
+
+#ifdef DEBUG_TASKGRAPH
+ TGDBG("worklist length: %d\n", worklist_length);
+
+ __kmp_taskgraph_region_dot(entryregion, "PredsAndSuccs");
+#endif
+
+ kmp_taskgraph_region_t **order =
+ (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+ thread, worklist_length * sizeof(kmp_taskgraph_region_t *));
+ kmp_taskgraph_region_t **doms =
+ (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+ thread, worklist_length * sizeof(kmp_taskgraph_region_t *));
+ memset(doms, 0, worklist_length * sizeof(kmp_taskgraph_region_t *));
+ kmp_int32 cursor = worklist_length;
+ assert(entryregion->type == TASKGRAPH_REGION_ENTRY);
+ __kmp_taskgraph_region_dfs(entryregion, order, cursor, false);
+ assert(cursor == 0);
+ __kmp_taskgraph_region_doms(order, doms, worklist_length, false);
+
+ bool dropped_preds_p = false;
+
+ for (kmp_int32 i = 0; i < worklist_length; i++) {
+ kmp_taskgraph_region_t *region = order[i];
+ if (__kmp_region_deplist_len(region->predecessors) < 2)
+ continue;
+ TGDBG("checking region %p for redundant predecessors\n", region);
+ kmp_taskgraph_region_dep_t **predp = ®ion->predecessors;
+ while (*predp) {
+ kmp_taskgraph_region_dep_t *pred = *predp;
+
+ bool passes_pred = false;
+ for (kmp_taskgraph_region_dep_t *rest = region->predecessors; rest;
+ rest = rest->next) {
+ if (rest->region == pred->region)
+ continue;
+ kmp_taskgraph_region_t *dom = doms[rest->region->timestamp];
+ TGDBG("pred region: %p, next: %p\n", pred->region, rest->region);
+ while (true) {
+ TGDBG("check against dom: %p\n", dom);
+ if (dom == pred->region) {
+ passes_pred = true;
+ break;
+ } else if (dom == doms[dom->timestamp]) {
+ break;
+ } else {
+ dom = doms[dom->timestamp];
+ }
+ }
+ if (passes_pred)
+ break;
+ }
+
+ if (passes_pred) {
+ // We can drop this predecessor.
+ TGDBG("dropping pred %p from region %p, dom %p\n", pred->region, region,
+ doms[pred->region->timestamp]);
+ kmp_taskgraph_region_dep_t *next = pred->next;
+ kmp_taskgraph_region_dep_t **succp = &pred->region->successors;
+ while (*succp) {
+ kmp_taskgraph_region_dep_t *succ = *succp;
+ if (succ->region == region) {
+ kmp_taskgraph_region_dep_t *nexts = succ->next;
+ __kmp_region_dep_recycle(&taskgraph->recycled_deps, succ);
+ *succp = nexts;
+ } else {
+ succp = &succ->next;
+ }
+ }
+ __kmp_region_dep_recycle(&taskgraph->recycled_deps, pred);
+ *predp = next;
+ dropped_preds_p = true;
+ } else {
+ predp = &pred->next;
+ }
+ }
+ }
+
+ if (dropped_preds_p) {
+ __kmp_fast_free(thread, order);
+ __kmp_fast_free(thread, doms);
+ return true;
+ }
+
+ kmp_bitset_t **pred_bitsets = nullptr;
+ kmp_bitset_t **succ_bitsets = nullptr;
+ kmp_uint64 *pred_sigs = nullptr;
+ kmp_uint64 *succ_sigs = nullptr;
+ kmp_int32 *bucket_head = nullptr;
+ kmp_int32 *bucket_next = nullptr;
+ kmp_uint32 bucket_mask = 0;
+
+ bool regions_combined_p = false;
+
+ struct kmp_dom_group_t {
+ kmp_taskgraph_region_t *dom;
+ kmp_int32 count;
+ };
+ kmp_dom_group_t *dom_groups =
+ (kmp_dom_group_t *)KMP_ALLOCA(worklist_length * sizeof(kmp_dom_group_t));
+
+ for (kmp_int32 i = 0; i < worklist_length; i++) {
+ kmp_taskgraph_region_t *region = order[i];
+ kmp_int32 num_groups = 0;
+ kmp_int32 npreds = __kmp_region_deplist_len(region->predecessors);
+ if (npreds >= 2) {
+ kmp_taskgraph_region_dep_t *pred;
+ for (pred = region->predecessors; pred; pred = pred->next) {
+ kmp_taskgraph_region_t *pred_region = pred->region;
+ kmp_taskgraph_region_t *this_dom = doms[pred_region->timestamp];
+ bool found = false;
+ for (kmp_int32 grp = 0; grp < num_groups; grp++) {
+ if (dom_groups[grp].dom == this_dom) {
+ dom_groups[grp].count++;
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ dom_groups[num_groups].dom = this_dom;
+ dom_groups[num_groups].count = 1;
+ num_groups++;
+ }
+ }
+
+ if (num_groups == 1 && region->mark != TASKGRAPH_COMBINED) {
+ TGDBG("region %p: all predecessors have a single dominator\n", region);
+
+ if (!pred_bitsets) {
+ pred_bitsets = (kmp_bitset_t **)__kmp_fast_allocate(
+ thread, sizeof(kmp_bitset_t *) * worklist_length);
+ succ_bitsets = (kmp_bitset_t **)__kmp_fast_allocate(
+ thread, sizeof(kmp_bitset_t *) * worklist_length);
+ pred_sigs = (kmp_uint64 *)__kmp_fast_allocate(
+ thread, sizeof(kmp_uint64) * worklist_length);
+ succ_sigs = (kmp_uint64 *)__kmp_fast_allocate(
+ thread, sizeof(kmp_uint64) * worklist_length);
+
+ for (kmp_int32 i = 0; i < worklist_length; i++) {
+ pred_bitsets[i] = __kmp_bitset_alloc(thread, worklist_length);
+ succ_bitsets[i] = __kmp_bitset_alloc(thread, worklist_length);
+ }
+
+ for (kmp_int32 j = 0; j < worklist_length; j++) {
+ kmp_taskgraph_region_t *reg = order[j];
+
+ for (pred = reg->predecessors; pred; pred = pred->next) {
+ __kmp_bitset_set(pred_bitsets[reg->timestamp],
+ pred->region->timestamp);
+ }
+
+ for (kmp_taskgraph_region_dep_t *succ = reg->successors; succ;
+ succ = succ->next) {
+ __kmp_bitset_set(succ_bitsets[reg->timestamp],
+ succ->region->timestamp);
+ }
+ }
+
+ for (kmp_int32 j = 0; j < worklist_length; j++) {
+ pred_sigs[j] = __kmp_bitset_signature(pred_bitsets[j]);
+ succ_sigs[j] = __kmp_bitset_signature(succ_bitsets[j]);
+ }
+
+ bucket_mask = 1;
+ while (bucket_mask < (kmp_uint32)worklist_length)
+ bucket_mask <<= 1;
+ bucket_mask -= 1;
+ bucket_head = (kmp_int32 *)__kmp_fast_allocate(
+ thread, sizeof(kmp_int32) * (bucket_mask + 1));
+ bucket_next = (kmp_int32 *)__kmp_fast_allocate(
+ thread, sizeof(kmp_int32) * worklist_length);
+ for (kmp_uint32 b = 0; b <= bucket_mask; b++)
+ bucket_head[b] = -1;
+ // Built back to front, so each chain lists its members in increasing
+ // order. That is what lets the scan below start at bucket_next[i]
+ // and see exactly the candidates after i, in the order it would have
+ // reached them by walking the whole array.
+ for (kmp_int32 j = worklist_length - 1; j >= 0; j--) {
+ kmp_uint32 b = __kmp_taskgraph_twin_bucket(
+ pred_sigs[j], succ_sigs[j], bucket_mask);
+ bucket_next[j] = bucket_head[b];
+ bucket_head[b] = j;
+ }
+ }
+
+ kmp_taskgraph_region_dep_t *equal_deps_chain = nullptr;
+
+ kmp_int32 same_preds_and_succs = 1;
+ bool any_mutex_p = __kmp_taskgraph_region_mutex_p(region);
+ // Twins share a signature and so share a bucket, and a bucket lists
+ // its members in increasing order, so starting at bucket_next[i]
+ // visits exactly the candidates worth comparing, rather than every
+ // remaining region. Equal buckets or equal signatures do not imply
+ // equal bitsets, so we still need to do the full __kmp_bitset_equal
+ // comparison when we find a potential match.
+ for (kmp_int32 j = bucket_next[i]; j >= 0; j = bucket_next[j]) {
+ if (pred_sigs[j] == pred_sigs[i] && succ_sigs[j] == succ_sigs[i] &&
+ order[j]->mark != TASKGRAPH_COMBINED &&
+ __kmp_bitset_equal(pred_bitsets[j], pred_bitsets[i]) &&
+ __kmp_bitset_equal(succ_bitsets[j], succ_bitsets[i])) {
+ TGDBG("regions %p and %p share all predecessors/successors\n",
+ order[i], order[j]);
+ same_preds_and_succs++;
+ equal_deps_chain = __kmp_region_deplist_add(
+ thread, &taskgraph->recycled_deps, order[j], equal_deps_chain);
+ if (__kmp_taskgraph_region_mutex_p(order[j]))
+ any_mutex_p = true;
+ }
+ }
+ if (same_preds_and_succs > 1) {
+ kmp_taskgraph_region_type region_type =
+ any_mutex_p ? TASKGRAPH_REGION_EXCLUSIVE
+ : TASKGRAPH_REGION_PARALLEL;
+ auto *par_region = __kmp_taskgraph_region_alloc(
+ thread, taskgraph, alloc_chain, region_type, same_preds_and_succs,
+ nullptr);
+ par_region->inner.children[0] = region;
+ region->mark = TASKGRAPH_COMBINED;
+ region->parent = par_region;
+ for (kmp_int32 j = 1; j < same_preds_and_succs; j++) {
+ kmp_taskgraph_region_dep_t *next = equal_deps_chain->next;
+ par_region->inner.children[j] = equal_deps_chain->region;
+ equal_deps_chain->region->mark = TASKGRAPH_COMBINED;
+ equal_deps_chain->region->parent = par_region;
+ __kmp_region_dep_recycle(&taskgraph->recycled_deps,
+ equal_deps_chain);
+ equal_deps_chain = next;
+ }
+ par_region->predecessors =
+ par_region->inner.children[0]->predecessors;
+ par_region->inner.children[0]->predecessors = nullptr;
+ par_region->successors = par_region->inner.children[0]->successors;
+ par_region->inner.children[0]->successors = nullptr;
+
+ // Redirect incoming deps to point to new parallel region.
+ for (pred = par_region->predecessors; pred; pred = pred->next) {
+ kmp_taskgraph_region_t *pred_region = pred->region;
+ kmp_taskgraph_region_dep_t **succp = &pred_region->successors;
+ while (*succp) {
+ kmp_taskgraph_region_dep_t *succ = *succp;
+ if (succ->region == par_region->inner.children[0]) {
+ succ->region = par_region;
+ succp = &succ->next;
+ } else {
+ bool found = false;
+ for (kmp_int32 j = 1; j < same_preds_and_succs; j++) {
+ if (succ->region == par_region->inner.children[j]) {
+ found = true;
+ break;
+ }
+ }
+ if (found) {
+ kmp_taskgraph_region_dep_t *next = succ->next;
+ __kmp_region_dep_recycle(&taskgraph->recycled_deps, succ);
+ *succp = next;
+ } else {
+ succp = &succ->next;
+ }
+ }
+ }
+ }
+
+ for (kmp_taskgraph_region_dep_t *succ = par_region->successors; succ;
+ succ = succ->next) {
+ kmp_taskgraph_region_t *succ_region = succ->region;
+ kmp_taskgraph_region_dep_t **predp = &succ_region->predecessors;
+ while (*predp) {
+ kmp_taskgraph_region_dep_t *pred = *predp;
+ if (pred->region == par_region->inner.children[0]) {
+ pred->region = par_region;
+ predp = &pred->next;
+ } else {
+ bool found = false;
+ for (kmp_int32 j = 1; j < same_preds_and_succs; j++) {
+ if (pred->region == par_region->inner.children[j]) {
+ found = true;
+ break;
+ }
+ }
+ if (found) {
+ kmp_taskgraph_region_dep_t *next = pred->next;
+ __kmp_region_dep_recycle(&taskgraph->recycled_deps, pred);
+ *predp = next;
+ } else {
+ predp = &pred->next;
+ }
+ }
+ }
+ }
+
+ par_region->next = region->next;
+ region->next = par_region;
+
+ regions_combined_p = true;
+ }
+ }
+ }
+ }
+
+ if (pred_bitsets) {
+ for (kmp_int32 j = 0; j < worklist_length; j++) {
+ __kmp_bitset_free(thread, pred_bitsets[j]);
+ __kmp_bitset_free(thread, succ_bitsets[j]);
+ }
+ __kmp_fast_free(thread, pred_bitsets);
+ __kmp_fast_free(thread, succ_bitsets);
+ __kmp_fast_free(thread, pred_sigs);
+ __kmp_fast_free(thread, succ_sigs);
+ __kmp_fast_free(thread, bucket_head);
+ __kmp_fast_free(thread, bucket_next);
+ }
+
+ // Case 2 made progress: let the build loop re-run the series/parallel
+ // collapse before we look at irreducibility again.
+ if (regions_combined_p) {
+ __kmp_fast_free(thread, order);
+ __kmp_fast_free(thread, doms);
+ return true;
+ }
+
+ // Case 3: neither the transitive drop nor the twin merge applied, so the
+ // residual is genuinely irreducible. Decompose it into nested SESE regions
+ // and carve only the minimal (leaf) irreducible ones, leaving any reducible
+ // structure (and its reductions) around the knots for the series/parallel
+ // collapse to pick up on the next pass. Each carve replaces a knot with a
+ // single IRREDUCIBLE atom (fewer live regions), so the build loop makes
+ // monotone progress and terminates.
+ [[maybe_unused]] bool carved = __kmp_taskgraph_carve_sese(
+ thread, taskgraph, alloc_chain, order, doms, worklist_length, exitregion);
+
+ assert(carved);
+
+ __kmp_fast_free(thread, order);
+ __kmp_fast_free(thread, doms);
+ return true;
+}
+
+/// Build a nested region structure out of a recorded taskgraph.
+//
+// The algorithm proceeds by alternating two phases until a single top-level
+// node is reached. Briefly, and glossing over some details:
+//
+// 1. Serial-parallel decomposition. Chains of single-successor,
+// single-predecessor nodes are collapsed into a "sequential" region, and
+// nodes with >1 predecessor, where each predecessor has a single
+// predecessor and a single successor, are collapsed into "parallel" regions.
+//
+// 2. Irreducible-graph processing. Several techniques are used to turn graphs
+// not handled by step (1) into graphs that can be handled by that step.
+//
+// Notably, simple graphs that can be handled entirely by step (1) avoid doing
+// much of the heavier processing involved in step (2), so the common case
+// should be relatively fast.
+
+static kmp_taskgraph_region_t *__kmp_taskgraph_build_regions(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t *entryregion,
+ kmp_taskgraph_region_t *exitregion) {
+ bool changed;
+ kmp_int32 phase = 0;
+
+#ifdef DEBUG_TASKGRAPH
+ __kmp_taskgraph_region_dot(entryregion, "InitialPredsAndSuccs");
+#endif
+
+ __kmp_taskgraph_region_chain_clear_marks(entryregion);
+
+ while (true) {
+ do {
+ changed = false;
+ TGDBG("starting seq pass\n");
+ for (kmp_taskgraph_region_t **seq_head = &entryregion; *seq_head;
+ seq_head = &(*seq_head)->next) {
+ TGDBG("consider %s region: %p\n",
+ __kmp_taskgraph_region_type_name((*seq_head)->type), *seq_head);
+ if ((*seq_head)->mark == TASKGRAPH_COMBINED) {
+ TGDBG("already combined\n");
+ continue;
+ }
+ changed |= __kmp_taskgraph_collapse_sequence(thread, taskgraph,
+ alloc_chain, seq_head,
+ /*parent=*/nullptr, phase);
+ TGDBG("changed: %s\n", changed ? "true" : "false");
+ }
+ ++phase;
+ __kmp_taskgraph_region_chain_prune(&entryregion);
+ __kmp_taskgraph_region_worklist_check(thread, taskgraph, entryregion,
+ "after seq collapse");
+ TGDBG("starting par/unordered pass\n");
+ for (kmp_taskgraph_region_t **par_head = &entryregion; *par_head;
+ par_head = &(*par_head)->next) {
+ TGDBG("consider %s region: %p\n",
+ __kmp_taskgraph_region_type_name((*par_head)->type), *par_head);
+ if ((*par_head)->mark == TASKGRAPH_COMBINED) {
+ TGDBG("already combined\n");
+ continue;
+ }
+ changed |= __kmp_taskgraph_collapse_par_exclusive(
+ thread, taskgraph, alloc_chain, par_head, /*parent=*/nullptr,
+ phase);
+ TGDBG("changed: %s\n", changed ? "true" : "false");
+ }
+ ++phase;
+ __kmp_taskgraph_region_chain_prune(&entryregion);
+ __kmp_taskgraph_region_worklist_check(thread, taskgraph, entryregion,
+ "after par collapse");
+ } while (changed);
+
+ if (entryregion->type == TASKGRAPH_REGION_ENTRY) {
+ if (__kmp_region_deplist_len(entryregion->successors) == 1) {
+ kmp_taskgraph_region_t *one_region = entryregion->successors->region;
+ if (__kmp_region_deplist_len(one_region->successors) == 1) {
+ kmp_taskgraph_region_t *maybe_exit = one_region->successors->region;
+ if (maybe_exit->type == TASKGRAPH_REGION_EXIT)
+ return one_region;
+ }
+ }
+ } else {
+ fprintf(stderr, "FIXME: Expected entry region!\n");
+ return entryregion;
+ }
+
+ TGDBG("attempting to collapse irreducible regions\n");
+
+ changed |= __kmp_taskgraph_rewrite_irreducible(
+ thread, taskgraph, alloc_chain, &entryregion, exitregion);
+
+ // rewrite_irreducible always makes progress now: it either applies a safe
+ // reduction (transitive drop / twin merge) or carves a subgraph of the
+ // residual into a single IRREDUCIBLE region, so the loop is guaranteed to
+ // terminate.
+ KMP_DEBUG_ASSERT(changed);
+ if (!changed)
+ return entryregion;
+ }
+
+ return entryregion;
+}
+
+/// Give every container region the union of its children's mutex sets, so that
+/// __kmp_taskgraph_find_exclusive_regions can ask what the members of an
+/// EXCLUSIVE region hold between them. BITSIZE is the number of mutex sets in
+/// the graph, i.e. the width of the sets built here.
+
+static void __kmp_taskgraph_gather_mutex_sets(kmp_info_t *thread,
+ kmp_taskgraph_region_t *region,
+ kmp_size_t bitsize) {
+ switch (region->type) {
+ case TASKGRAPH_REGION_ENTRY:
+ case TASKGRAPH_REGION_EXIT:
+ case TASKGRAPH_REGION_WAIT:
+ case TASKGRAPH_REGION_NODE:
+ // A task node already carries its recorded set membership, and has no
+ // children to recurse into.
+ return;
+ // Every kind of container folds the same way: what its members hold between
+ // them does not depend on how it orders them, only on which of them there
+ // are.
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_PARALLEL:
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ case TASKGRAPH_REGION_IRREDUCIBLE: {
+ KMP_DEBUG_ASSERT(!region->mutexset);
+ kmp_bitset_t *combined = __kmp_bitset_alloc(thread, bitsize);
+ __kmp_bitset_clearall(combined);
+ for (kmp_int32 child = 0; child < region->inner.num_children; child++) {
+ __kmp_taskgraph_gather_mutex_sets(thread, region->inner.children[child],
+ bitsize);
+ if (region->inner.children[child]->mutexset)
+ __kmp_bitset_or(combined, combined,
+ region->inner.children[child]->mutexset);
+ }
+ region->mutexset = combined;
+ return;
+ }
+ }
+}
+
+static int __kmp_popcount_cmp(const void *a, const void *b) {
+ const kmp_taskgraph_region_t *reg_a = *(kmp_taskgraph_region_t **)a;
+ const kmp_taskgraph_region_t *reg_b = *(kmp_taskgraph_region_t **)b;
+ kmp_int32 popc_a = 0, popc_b = 0;
+ if (reg_a->mutexset)
+ popc_a = __kmp_bitset_popcount(reg_a->mutexset);
+ if (reg_b->mutexset)
+ popc_b = __kmp_bitset_popcount(reg_b->mutexset);
+ if (popc_a > popc_b)
+ return -1;
+ else if (popc_a < popc_b)
+ return 1;
+ return 0;
+}
+
+/// Find "mutexinoutset" regions that can be represented without explicit
+// mutexes, i.e. using "TASKGRAPH_REGION_EXCLUSIVE".
+
+static void __kmp_taskgraph_find_exclusive_regions(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t **region_p) {
+ kmp_taskgraph_region_t *region = *region_p;
+ switch (region->type) {
+ case TASKGRAPH_REGION_ENTRY:
+ case TASKGRAPH_REGION_EXIT:
+ case TASKGRAPH_REGION_NODE:
+ case TASKGRAPH_REGION_WAIT:
+ break;
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_PARALLEL:
+ // A carved irreducible region is a structural container: it has no mutex set
+ // of its own, but its children may, so recurse like the other containers.
+ case TASKGRAPH_REGION_IRREDUCIBLE: {
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ __kmp_taskgraph_find_exclusive_regions(thread, taskgraph, alloc_chain,
+ ®ion->inner.children[c]);
+ }
+ break;
+ }
+ case TASKGRAPH_REGION_EXCLUSIVE: {
+ qsort(region->inner.children, region->inner.num_children,
+ sizeof(kmp_taskgraph_region_t *), __kmp_popcount_cmp);
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ TGDBG("building tree: region mutexset = 0x%llx\n",
+ (unsigned long long)region->inner.children[c]->mutexset
+ ? region->inner.children[c]->mutexset->bits[0]
+ : 0);
+ region->inner.children[c]->mark = TASKGRAPH_UNMARKED;
+ }
+ kmp_bitset_t *conflicts =
+ __kmp_bitset_alloc(thread, region->mutexset->bitsize);
+ kmp_bitset_t *subsets_cover =
+ __kmp_bitset_alloc(thread, region->mutexset->bitsize);
+ __kmp_bitset_copy(conflicts, region->mutexset);
+ bool irregular = false;
+ kmp_int32 combined_children = 0;
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ kmp_bitset_t *candidate = region->inner.children[c]->mutexset;
+ if (__kmp_bitset_empty_p(candidate))
+ continue;
+ __kmp_bitset_clearall(subsets_cover);
+ bool found_subset = false;
+ bool other_overlaps = false;
+ for (kmp_int32 d = c + 1; d < region->inner.num_children; d++) {
+ // This could test for a subset in some cases, but that adds
+ // complication for later processing. Maybe revisit later if it
+ // seems worthwhile.
+ // E.g. if we have deps like this:
+ //
+ // #pragma omp task depend(mutexinoutset: deps[0], deps[1]) { /*a*/ }
+ // #pragma omp task depend(mutexinoutset: deps[0]) { /*b*/ }
+ // #pragma omp task depend(mutexinoutset: deps[1]) { /*c*/ }
+ //
+ // This could be represented as:
+ //
+ // exclusive {
+ // node: a
+ // parallel {
+ // node: b
+ // node: c
+ // }
+ // }
+ //
+ // We're not doing that yet though.
+ if (__kmp_bitset_equal(candidate,
+ region->inner.children[d]->mutexset)) {
+ found_subset = true;
+ __kmp_bitset_or(subsets_cover, subsets_cover,
+ region->inner.children[d]->mutexset);
+ } else if (__kmp_bitset_intersect_p(
+ candidate, region->inner.children[d]->mutexset)) {
+ other_overlaps = true;
+ break;
+ }
+ }
+ if (!found_subset || other_overlaps)
+ continue;
+ if (!__kmp_bitset_equal(subsets_cover, candidate)) {
+ TGDBG("subsets cover: 0x%llx, candidate: 0x%llx\n",
+ (unsigned long long)subsets_cover->bits[0],
+ (unsigned long long)candidate->bits[0]);
+ irregular = true;
+ break;
+ }
+ for (kmp_int32 d = c + 1; d < region->inner.num_children; d++) {
+ if (region->inner.children[d]->mutexset_parent)
+ continue;
+ // As above wrt. subsets.
+ if (__kmp_bitset_equal(candidate,
+ region->inner.children[d]->mutexset)) {
+ TGDBG("set index %d's parent to index %d\n", d, c);
+ region->inner.children[d]->mutexset_parent =
+ region->inner.children[c];
+ combined_children++;
+ __kmp_bitset_and_not(conflicts, conflicts, candidate);
+ }
+ }
+ }
+ TGDBG("irregular: %s\n", irregular ? "true" : "false");
+ TGDBG("final conflicts: 0x%llx\n", (unsigned long long)conflicts->bits[0]);
+ __kmp_bitset_free(thread, subsets_cover);
+ region->type = TASKGRAPH_REGION_PARALLEL;
+ if (!irregular && __kmp_bitset_empty_p(conflicts)) {
+ TGDBG("transforming exclusive region %p\n", region);
+ TGDBG("orig region children: %d\n", region->inner.num_children);
+ TGDBG("combined children: %d\n", combined_children);
+ if (region->inner.num_children == combined_children + 1) {
+ region->type = TASKGRAPH_REGION_EXCLUSIVE;
+ } else {
+ kmp_int32 par_elems = region->inner.num_children - combined_children;
+ auto *new_par = __kmp_taskgraph_region_alloc(
+ thread, taskgraph, alloc_chain, TASKGRAPH_REGION_PARALLEL,
+ par_elems, nullptr);
+ for (kmp_int32 c = region->inner.num_children - 1; c >= 0; c--) {
+ kmp_taskgraph_region_t *child = region->inner.children[c];
+ // Make mutex set into a circular list.
+ if (child->mutexset_parent && child->mark != TASKGRAPH_TEMP_MARK) {
+ if (!child->mutexset_parent->mutexset_parent) {
+ // child <-> parent
+ child->mutexset_parent->mutexset_parent = child;
+ child->mutexset_parent->mark = TASKGRAPH_TEMP_MARK;
+ } else {
+ kmp_taskgraph_region_t *parent = child->mutexset_parent;
+ child->mutexset_parent = parent->mutexset_parent;
+ parent->mutexset_parent = child;
+ parent->mark = TASKGRAPH_TEMP_MARK;
+ }
+ }
+ }
+ kmp_int32 idx = 0;
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ kmp_taskgraph_region_t *child = region->inner.children[c];
+ TGDBG("process child: %p\n", child);
+ if (child->mutexset_parent && child->mark != TASKGRAPH_COMBINED) {
+ kmp_int32 elems = 0;
+ kmp_taskgraph_region_t *next = child;
+ do {
+ elems++;
+ next = next->mutexset_parent;
+ } while (next != child);
+ TGDBG("make exclusive region with %d children\n", elems);
+ auto *excl_region = __kmp_taskgraph_region_alloc(
+ thread, taskgraph, alloc_chain, TASKGRAPH_REGION_EXCLUSIVE,
+ elems, nullptr);
+ kmp_int32 excl_child = 0;
+ next = child;
+ do {
+ excl_region->inner.children[excl_child++] = next;
+ next->mark = TASKGRAPH_COMBINED;
+ next = next->mutexset_parent;
+ } while (next != child);
+ assert(excl_child == excl_region->inner.num_children);
+ new_par->inner.children[idx++] = excl_region;
+ } else if (!child->mutexset_parent) {
+ new_par->inner.children[idx++] = child;
+ }
+ }
+ TGDBG("idx=%d, supposed to be %d\n", idx, new_par->inner.num_children);
+ assert(idx == new_par->inner.num_children);
+ *region_p = new_par;
+ region->mark = TASKGRAPH_DELETED;
+ }
+ }
+ __kmp_bitset_free(thread, conflicts);
+ break;
+ }
+ default:
+ assert(false && "unreachable");
+ }
+}
+
+/// Strip mutex sets from taskgraph region, except those needed at runtime.
+
+static kmp_int32
+__kmp_taskgraph_strip_mutex_sets(kmp_info_t *thread,
+ kmp_taskgraph_region_t *region,
+ bool in_exclusive = false) {
+ kmp_int32 mutexes_needed = 0;
+ switch (region->type) {
+ case TASKGRAPH_REGION_ENTRY:
+ case TASKGRAPH_REGION_EXIT:
+ case TASKGRAPH_REGION_WAIT:
+ assert(!region->mutexset);
+ break;
+ case TASKGRAPH_REGION_NODE:
+ if (region->mutexset) {
+ if (in_exclusive) {
+ __kmp_bitset_free(thread, region->mutexset);
+ region->mutexset = nullptr;
+ } else {
+ // FIXME: This might be pessimistic -- the remaining mutex sets might
+ // have holes or duplicates. We could compact them.
+ kmp_int32 m = region->mutexset->bitsize;
+ mutexes_needed = std::max(mutexes_needed, m);
+ }
+ }
+ break;
+ case TASKGRAPH_REGION_EXCLUSIVE: {
+ if (region->mutexset) {
+ __kmp_bitset_free(thread, region->mutexset);
+ region->mutexset = nullptr;
+ }
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ kmp_int32 m = __kmp_taskgraph_strip_mutex_sets(
+ thread, region->inner.children[c], true);
+ mutexes_needed = std::max(mutexes_needed, m);
+ }
+ break;
+ }
+ default: {
+ if (region->mutexset) {
+ __kmp_bitset_free(thread, region->mutexset);
+ region->mutexset = nullptr;
+ }
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ kmp_int32 m = __kmp_taskgraph_strip_mutex_sets(
+ thread, region->inner.children[c], in_exclusive);
+ mutexes_needed = std::max(mutexes_needed, m);
+ }
+ }
+ }
+ return mutexes_needed;
+}
+
+static void __kmp_taskgraph_exclusive_regions(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t **region_p,
+ kmp_int32 max_mutex) {
+ // No dependence in the graph was a mutexinoutset, so no node carries a mutex
+ // set and there is nothing here to do. The gather would give each container
+ // an empty set only for the strip to free it again; no EXCLUSIVE region can
+ // exist to be rewritten, since one is built only where
+ // __kmp_taskgraph_region_mutex_p holds and that needs a node set; and the
+ // mutex count comes out zero either way. Skipping is what keeps recording a
+ // wide graph off the gather's per-container walk.
+ if (max_mutex == 0) {
+ taskgraph->num_mutexes = 0;
+ return;
+ }
+ __kmp_taskgraph_gather_mutex_sets(thread, *region_p, max_mutex);
+ __kmp_taskgraph_find_exclusive_regions(thread, taskgraph, alloc_chain,
+ region_p);
+ taskgraph->num_mutexes = __kmp_taskgraph_strip_mutex_sets(thread, *region_p);
+}
+
+static const char *
+__kmp_taskgraph_region_type_name(kmp_taskgraph_region_type type) {
+ switch (type) {
+ case TASKGRAPH_REGION_ENTRY:
+ return "entry";
+ case TASKGRAPH_REGION_EXIT:
+ return "exit";
+ case TASKGRAPH_REGION_NODE:
+ return "node";
+ case TASKGRAPH_REGION_WAIT:
+ return "wait";
+ case TASKGRAPH_REGION_PARALLEL:
+ return "parallel";
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ return "exclusive";
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ return "sequential";
+ case TASKGRAPH_REGION_IRREDUCIBLE:
+ return "irreducible";
+ default:
+ return "<unknown>";
+ }
+}
+
+static void __kmp_dump_taskgraph_regions(FILE *f,
+ kmp_taskgraph_region_t *region,
+ int indent = 0) {
+ switch (region->type) {
+ case TASKGRAPH_REGION_ENTRY:
+ case TASKGRAPH_REGION_EXIT:
+ fprintf(f, "%*s%s node\n", indent, "",
+ __kmp_taskgraph_region_type_name(region->type));
+ break;
+ case TASKGRAPH_REGION_NODE:
+ case TASKGRAPH_REGION_WAIT: {
+ char set_membership[40];
+ if (region->mutexset)
+ sprintf(set_membership, " [sets: 0x%llx]",
+ (unsigned long long)region->mutexset->bits[0]);
+ else
+ strcpy(set_membership, "");
+ fprintf(f, "%*s%s: %p%s\n", indent, "",
+ __kmp_taskgraph_region_type_name(region->type), region->task.node,
+ set_membership);
+ break;
+ }
+ default: {
+ char set_membership[40];
+ if (region->mutexset)
+ sprintf(set_membership, " [sets: 0x%llx]",
+ (unsigned long long)region->mutexset->bits[0]);
+ else
+ strcpy(set_membership, "");
+ fprintf(f, "%*s%s%s {\n", indent, "",
+ __kmp_taskgraph_region_type_name(region->type), set_membership);
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ __kmp_dump_taskgraph_regions(f, region->inner.children[c], indent + 2);
+ }
+ fprintf(f, "%*s}\n", indent, "");
+ }
+ }
+}
+
+#ifdef DEBUG_TASKGRAPH
+
+static kmp_taskgraph_region_dep_t *
+__kmp_dump_find_parent_regions(kmp_info *thd, kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t *region, int numregions,
+ kmp_taskgraph_region_dep_t *list = nullptr) {
+ for (int r = 0; r < numregions; r++) {
+ if (!region[r].parent)
+ continue;
+ bool in_list = false;
+ for (kmp_taskgraph_region_dep_t *dep = list; dep; dep = dep->next) {
+ if (dep->region == region[r].parent) {
+ in_list = true;
+ break;
+ }
+ }
+ if (!in_list) {
+ list = __kmp_region_deplist_add(thd, &taskgraph->recycled_deps,
+ region[r].parent, list);
+ list = __kmp_dump_find_parent_regions(thd, taskgraph, region[r].parent, 1,
+ list);
+ }
+ }
+ return list;
+}
+
+static void __kmp_dump_raw_taskgraph_regions(FILE *f, kmp_info *thd,
+ kmp_taskgraph_record_t *taskgraph,
+ kmp_taskgraph_region_t *region,
+ int numregions, int indent = 0) {
+ kmp_taskgraph_region_dep_t *parentlist = nullptr;
+ kmp_taskgraph_region_dep_t *printedlist = nullptr;
+ for (int r = 0; r < numregions; r++) {
+ int children = 0;
+ if (region[r].type == TASKGRAPH_REGION_PARALLEL ||
+ region[r].type == TASKGRAPH_REGION_SEQUENTIAL ||
+ region[r].type == TASKGRAPH_REGION_EXCLUSIVE ||
+ region[r].type == TASKGRAPH_REGION_IRREDUCIBLE)
+ children = region[r].inner.num_children;
+ fprintf(
+ f,
+ "%*sregion %d (%p): %s%s (%d children) parent %p succs %d preds %d\n",
+ indent, "", r, ®ion[r],
+ __kmp_taskgraph_region_type_name(region[r].type),
+ region[r].mark == TASKGRAPH_COMBINED ? " (combined)" : "", children,
+ region[r].parent, __kmp_region_deplist_len(region[r].successors),
+ __kmp_region_deplist_len(region[r].predecessors));
+ if (children > 0) {
+ for (int c = 0; c < children; c++)
+ __kmp_dump_raw_taskgraph_regions(
+ f, thd, taskgraph, region->inner.children[c], 1, indent + 2);
+ }
+ }
+ if (indent == 0) {
+ parentlist =
+ __kmp_dump_find_parent_regions(thd, taskgraph, region, numregions);
+ fprintf(stderr, "%*sfound %d parent region(s):\n", indent, "",
+ __kmp_region_deplist_len(parentlist));
+ for (kmp_taskgraph_region_dep_t *p = parentlist; p; p = p->next) {
+ __kmp_dump_raw_taskgraph_regions(f, thd, taskgraph, p->region, 1,
+ indent + 2);
+ }
+ __kmp_region_deplist_recycle(&taskgraph->recycled_deps, parentlist);
+ }
+}
+#endif
+
+bool __kmp_taskgraph_trace() {
+ static int trace_enabled = -1;
+ if (trace_enabled == -1) {
+ trace_enabled = getenv("KMP_TASKGRAPH_TRACE") != nullptr;
+ }
+ return trace_enabled;
+}
+
+bool __kmp_taskgraph_check_lifetime() {
+ static int check_enabled = -1;
+ if (check_enabled == -1) {
+ check_enabled = getenv("KMP_TASKGRAPH_CHECK_LIFETIME") != nullptr;
+ }
+ return check_enabled;
+}
+
+/// Build a nested region structure from a "raw" recorded taskgraph, and mark
+/// the taskgraph ready for replay.
+//
+// The input to this function consists of tasks with *data* dependencies
+// between them. The output of the function is a nested tree structure: the
+// dependencies between tasks implicitly become *control* dependencies. In
+// the common case, these ought to map straightforwardly to hardware-provided
+// execution primitives (e.g. on a GPU), or to runtime-provided primitives (for
+// the CPU).
+//
+// Here is an example taskgraph:
+//
+// #pragma omp taskgraph
+// {
+// #pragma omp task depend(out: deps[2])
+// { }
+// #pragma omp task depend(out: deps[0], deps[1])
+// { }
+// #pragma omp task depend(inout: deps[0])
+// { }
+// #pragma omp task depend(inout: deps[1])
+// { }
+// #pragma omp task depend(inout: deps[2])
+// { }
+// #pragma omp task depend(in: deps[0], deps[1], deps[2])
+// { }
+// }
+//
+// This dependency graph is "reducible", and the resulting tree looks like this:
+//
+// sequential {
+// parallel {
+// sequential {
+// node: 0x588aa11021b0
+// node: 0x588aa1102250
+// }
+// sequential {
+// node: 0x588aa11021d8
+// parallel {
+// node: 0x588aa1102228
+// node: 0x588aa1102200
+// }
+// }
+// }
+// node: 0x588aa1102278
+// }
+//
+// Each node represents a task, and the containing parallel and sequential
+// regions represent sub-regions that can be executed in parallel, or
+// one-at-a-time, in order.
+//
+// In some cases, the data-dependency graph may not be trivially reducible to
+// parallel and sequential regions. In this case, several techniques are used
+// to produce a reducible graph from an irreducible graph (see
+// __kmp_taskgraph_rewrite_irreducible).
+//
+// For example in this graph:
+//
+// #pragma omp taskgraph
+// {
+// #pragma omp task depend(out: deps[0], deps[1])
+// { }
+// #pragma omp task depend(out: deps[2], deps[3])
+// { }
+// #pragma omp task depend(inout: deps[0])
+// { }
+// #pragma omp task depend(inout: deps[1])
+// { }
+// #pragma omp task depend(inout: deps[2])
+// { }
+// #pragma omp task depend(inout: deps[3])
+// { }
+// #pragma omp task depend(in: deps[0], deps[1], deps[2], deps[3])
+// { }
+// #pragma omp task depend(in: deps[1], deps[2])
+// { }
+// }
+//
+// The final two tasks overlap data dependencies in such a way that the
+// resulting dependency graph cannot be trivially decomposed to parallel and
+// sequential regions. In this case the whole graph is reduced to an
+// irreducible region:
+//
+// irreducible {
+// node: 0x7803e00fb8d8
+// node: 0x7803e00fb8a8
+// node: 0x7803e00fb848
+// node: 0x7803e00fb938
+// node: 0x7803e00fb998
+// node: 0x7803e00fb968
+// node: 0x7803e00fb908
+// node: 0x7803e00fb878
+// }
+//
+// If *part* of the graph is reducible, it should still be handled without
+// falling back to an irreducible region. If we chnage the input as follows:
+//
+// #pragma omp taskgraph
+// {
+// #pragma omp task depend(out: deps[0], deps[1])
+// { }
+// #pragma omp task depend(out: deps[2], deps[3])
+// { }
+// #pragma omp task depend(inout: deps[0])
+// { }
+// #pragma omp task depend(inout: deps[1])
+// { }
+// #pragma omp task depend(inout: deps[2])
+// { }
+// #pragma omp task depend(inout: deps[3])
+// { }
+// #pragma omp task depend(in: deps[0], deps[1], deps[2], deps[3])
+// { }
+// #pragma omp task depend(in: deps[1], deps[2])
+// { }
+// int res = 0;
+// #pragma omp taskloop reduction(+: res) num_tasks(4)
+// {
+// for (int i = 0; i < 16; i++)
+// res += i;
+// }
+// }
+//
+// We will now see:
+//
+// irreducible {
+// node: 0x58de797a9328
+// node: 0x58de797a92f8
+// node: 0x58de797a9298
+// node: 0x58de797a9388
+// node: 0x58de797a93e8
+// node: 0x58de797a93b8
+// node: 0x58de797a9358
+// node: 0x58de797a92c8
+// sequential {
+// parallel {
+// node: 0x58de797a94a8
+// node: 0x58de797a9478
+// node: 0x58de797a9448
+// node: 0x58de797a9418
+// }
+// wait: 0x58de797a94d8
+// }
+// }
+//
+// This is particularly important for reductions, which won't work with the
+// "unstructured" irreducible region.
+//
+// The final region type is "exclusive", which arises for "mutexinoutset"
+// dependencies that are able to be abstracted away (we can't do this in all
+// cases: when we can't, we still use explicit mutexes).
+//
+// An example of this:
+//
+// #pragma omp taskgraph
+// {
+// #pragma omp task depend(mutexinoutset: deps[0])
+// { }
+// #pragma omp task depend(mutexinoutset: deps[1])
+// { }
+// #pragma omp task depend(mutexinoutset: deps[0])
+// { }
+// #pragma omp task depend(mutexinoutset: deps[1])
+// { }
+// #pragma omp task depend(mutexinoutset: deps[0])
+// { }
+// #pragma omp task depend(mutexinoutset: deps[1])
+// { }
+// #pragma omp task depend(mutexinoutset: deps[0])
+// { }
+// #pragma omp task depend(mutexinoutset: deps[1])
+// { }
+// }
+//
+// Results in this structure:
+//
+// parallel {
+// exclusive {
+// node: 0x5c0c5c571120
+// node: 0x5c0c5c5710d0
+// node: 0x5c0c5c571080
+// node: 0x5c0c5c571030
+// }
+// exclusive {
+// node: 0x5c0c5c5710f8
+// node: 0x5c0c5c5710a8
+// node: 0x5c0c5c571058
+// node: 0x5c0c5c571008
+// }
+// }
+//
+// The meaning of "exclusive" here is for each of the child regions (task
+// nodes in this case) to be executed in some unspecified order, one at a
+// time relative to the other regions in the structure. E.g. a GPU
+// implementation could try to dynamically schedule tasks such that they fit
+// instantaneously-available execution resources.
+//
+// In cases where mutexes cannot be abstracted, each affected task node is
+// annotated with a set of mutexes that must be held while executing the task.
+// (Shown with [sets: 0xN] in dump output).
+
+kmp_int32 __kmp_build_taskgraph(kmp_int32 gtid,
+ kmp_taskdata_t *current_taskdata,
+ kmp_taskgraph_record_t *taskgraph) {
+ kmp_int32 numnodes = taskgraph->num_tasks;
+ kmp_int32 numregions = numnodes + 2;
+ kmp_taskgraph_node_t *nodes = taskgraph->record_map;
+ kmp_info_t *thread = __kmp_threads[gtid];
+ bool dep_barrier = false;
+
+ // An empty taskgraph isn't useful and causes analysis problems below, so
+ // catch it early.
+ if (numnodes == 0) {
+ taskgraph->root = nullptr;
+ taskgraph->alloc_root = nullptr;
+ KMP_ATOMIC_ST_REL(&taskgraph->status, KMP_TDG_READY);
+ return 0;
+ }
+
+ kmp_dephash_t *hash = __kmp_dephash_create(thread, current_taskdata);
+
+ // We need to take special care to align the all_depnodes array to the cache
+ // line size, because kmp_depnode_t is marked as 64-byte aligned and
+ // otherwise the compiler might generate faulting memory accesses based on
+ // that alignment assumption.
+ size_t all_depnodes_size = numregions * sizeof(kmp_depnode_t);
+ // The maximum amount of padding we need is CACHE_LINE - 1 bytes.
+ all_depnodes_size = all_depnodes_size + CACHE_LINE - 1;
+ char *all_depnodes_misaligned =
+ (char *)__kmp_thread_malloc(thread, all_depnodes_size);
+ kmp_depnode_t *all_depnodes =
+ (kmp_depnode_t *)((((intptr_t)all_depnodes_misaligned) + CACHE_LINE - 1) &
+ ~(CACHE_LINE - 1));
+ kmp_int32 next_mutex_set = 0;
+
+ for (kmp_int32 i = 0; i < numnodes; i++) {
+ int n_mtxs = 0;
+ bool dep_all;
+
+ dep_all = __kmp_filter_aliased_deps(nodes[i].ndeps, nodes[i].dep_list,
+ nodes[i].task, &n_mtxs);
+ kmp_depnode_t *node = &all_depnodes[i];
+ __kmp_init_node(node, /*on_stack=*/false);
+ node->dn.task = nodes[i].task;
+ dep_barrier = !nodes[i].task && nodes[i].taskloop_task;
+ if (!dep_all) {
+ __kmp_process_deps<taskgraph_deps>(gtid, node, &hash, dep_barrier,
+ nodes[i].ndeps, nodes[i].dep_list,
+ nodes[i].task, next_mutex_set);
+ } else {
+ __kmp_process_dep_all<taskgraph_deps>(gtid, node, hash, dep_barrier,
+ nodes[i].task);
+ }
+ }
+
+ kmp_taskgraph_region_t **order_out = (kmp_taskgraph_region_t **)KMP_ALLOCA(
+ numregions * sizeof(kmp_taskgraph_region_t *));
+ kmp_int32 outidx = 0;
+
+ kmp_taskgraph_region_t *initial_regions =
+ (kmp_taskgraph_region_t *)__kmp_fast_allocate(
+ thread, sizeof(kmp_taskgraph_region_t) * numregions);
+ memset(initial_regions, 0, sizeof(kmp_taskgraph_region_t) * numregions);
+
+ kmp_taskgraph_region_t *cfg_barrier = nullptr;
+
+ for (kmp_int32 i = 0; i < numnodes; i++) {
+ initial_regions[i].type =
+ nodes[i].task ? TASKGRAPH_REGION_NODE : TASKGRAPH_REGION_WAIT;
+ initial_regions[i].task.node = &nodes[i];
+ initial_regions[i].parent = nullptr;
+ if (i < numnodes - 1) {
+ initial_regions[i].next = &initial_regions[i + 1];
+ } else {
+ initial_regions[i].next = nullptr;
+ }
+ kmp_depnode_t *depnode = &all_depnodes[i];
+ initial_regions[i].mutexset = depnode->dn.set_membership;
+ for (kmp_depnode_list_t *succ = depnode->dn.successors; succ;
+ succ = succ->next) {
+ kmp_int32 succ_idx = succ->node - all_depnodes;
+ kmp_taskgraph_region_t *tg_succ = &initial_regions[succ_idx];
+ tg_succ->predecessors =
+ __kmp_region_deplist_add(thread, &taskgraph->recycled_deps,
+ &initial_regions[i], tg_succ->predecessors);
+ initial_regions[i].successors =
+ __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, tg_succ,
+ initial_regions[i].successors);
+ }
+ // Handle control flow dependencies. If a node (e.g. a taskloop task) has a
+ // wait after it corresponding to the end of an implicit taskgroup, join the
+ // task to the wait. The wait then becomes a barrier; any tasks after it
+ // will depend on the barrier.
+ if (nodes[i].cfg_successor != -1) {
+ kmp_int32 cfg_succ = nodes[i].cfg_successor;
+ initial_regions[i].successors = __kmp_region_deplist_add(
+ thread, &taskgraph->recycled_deps, &initial_regions[cfg_succ],
+ initial_regions[i].successors);
+ initial_regions[cfg_succ].predecessors = __kmp_region_deplist_add(
+ thread, &taskgraph->recycled_deps, &initial_regions[i],
+ initial_regions[cfg_succ].predecessors);
+ }
+ // Check for the TASKGRAPH_REGION_WAIT inserted after a taskloop.
+ bool is_taskgroup_marker = nodes[i].taskloop_task && !nodes[i].task;
+ // Avoid inserting duplicate edges.
+ if (cfg_barrier && !is_taskgroup_marker &&
+ !__kmp_region_deplist_contains(initial_regions[i].predecessors,
+ cfg_barrier)) {
+ cfg_barrier->successors = __kmp_region_deplist_add(
+ thread, &taskgraph->recycled_deps, &initial_regions[i],
+ cfg_barrier->successors);
+ initial_regions[i].predecessors = __kmp_region_deplist_add(
+ thread, &taskgraph->recycled_deps, cfg_barrier,
+ initial_regions[i].predecessors);
+ }
+ // If we find a post-taskloop WAIT or an undeferred task, record it as a
+ // barrier.
+ if (is_taskgroup_marker || nodes[i].undeferred)
+ cfg_barrier = &initial_regions[i];
+ }
+
+ __kmp_dephash_free<false>(thread, hash);
+
+ // We don't use the runtime refcounting mechanism for these depnodes, so
+ // free them explicitly here.
+ for (kmp_int32 i = 0; i < numnodes; i++) {
+ kmp_depnode_list_t *succ = all_depnodes[i].dn.successors;
+ while (succ) {
+ kmp_depnode_list_t *next = succ->next;
+ __kmp_fast_free(thread, succ);
+ succ = next;
+ }
+ all_depnodes[i].dn.successors = nullptr;
+ }
+
+ __kmp_thread_free(thread, all_depnodes_misaligned);
+
+ // We're done with the dep_list now.
+ for (kmp_int32 i = 0; i < numnodes; i++)
+ __kmp_thread_free(thread, nodes[i].dep_list);
+
+ // Use these indices for the virtual entry and exit regions
+ kmp_int32 entryregion = numnodes, exitregion = numnodes + 1;
+
+ // Set entry/exit node types, and add to worklist
+ initial_regions[entryregion].type = TASKGRAPH_REGION_ENTRY;
+ initial_regions[entryregion].next = &initial_regions[0];
+ initial_regions[exitregion].type = TASKGRAPH_REGION_EXIT;
+ initial_regions[numnodes - 1].next = &initial_regions[exitregion];
+
+ // Join entry and exit nodes up to the graph
+ for (kmp_int32 i = 0; i < numnodes; i++) {
+ kmp_taskgraph_region_t *region = &initial_regions[i];
+ kmp_int32 npreds = __kmp_region_deplist_len(region->predecessors);
+ kmp_int32 nsuccs = __kmp_region_deplist_len(region->successors);
+ if (npreds == 0) {
+ initial_regions[entryregion].successors =
+ __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, region,
+ initial_regions[entryregion].successors);
+ region->predecessors = __kmp_region_deplist_add(
+ thread, &taskgraph->recycled_deps, &initial_regions[entryregion],
+ region->predecessors);
+ }
+ if (nsuccs == 0) {
+ initial_regions[exitregion].predecessors =
+ __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, region,
+ initial_regions[exitregion].predecessors);
+ region->successors = __kmp_region_deplist_add(
+ thread, &taskgraph->recycled_deps, &initial_regions[exitregion],
+ region->successors);
+ }
+ region->owner = taskgraph;
+ }
+
+ kmp_int32 max_level = -1;
+
+ for (kmp_int32 i = 0; i < numregions; i++)
+ initial_regions[i].timestamp = i;
+
+ for (kmp_int32 i = 0; i < numregions; i++) {
+ if (initial_regions[i].mark == TASKGRAPH_UNMARKED) {
+ kmp_int32 level = __kmp_taskgraph_topological_order(&initial_regions[i],
+ order_out, &outidx);
+ max_level = level > max_level ? level : max_level;
+ }
+ }
+
+ assert(outidx == numregions);
+
+#ifdef DEBUG_TASKGRAPH
+ fprintf(stderr, "topological order (max level: %d):\n", max_level);
+
+ for (kmp_int32 i = 0; i < outidx; i++) {
+ fprintf(stderr, "node %d (region %p), level %d\n", order_out[i]->timestamp,
+ order_out[i], order_out[i]->level);
+ }
+#endif
+
+ kmp_taskgraph_region_t **alloc_chain = &initial_regions[0].alloc_chain;
+
+ kmp_taskgraph_region_t *root_region = __kmp_taskgraph_build_regions(
+ thread, taskgraph, alloc_chain, &initial_regions[entryregion],
+ &initial_regions[exitregion]);
+
+ __kmp_taskgraph_exclusive_regions(thread, taskgraph, alloc_chain,
+ &root_region, next_mutex_set);
+
+ *alloc_chain = nullptr;
+
+ taskgraph->root = root_region;
+ taskgraph->alloc_root = initial_regions;
+
+ // Free the dependency lists of the regions that made it into the tree.
+ __kmp_taskgraph_free_built_region_deps(thread, root_region);
+
+ // The entry and exit regions don't get added to the region tree, so free
+ // their dependency lists here.
+ __kmp_taskgraph_free_built_region_deps(thread, &initial_regions[entryregion]);
+ __kmp_taskgraph_free_built_region_deps(thread, &initial_regions[exitregion]);
+
+ // Free remaining regions that didn't end up in the region tree.
+ kmp_taskgraph_region_t **regp = &taskgraph->alloc_root;
+ while (*regp) {
+ kmp_taskgraph_region_t *reg = *regp;
+ if (reg->mark == TASKGRAPH_DELETED) {
+ kmp_taskgraph_region_t *chain_next = reg->alloc_chain;
+ TGDBG("deleted region from alloc chain: %p\n", reg);
+ __kmp_region_deplist_free(thread, reg->predecessors);
+ __kmp_region_deplist_free(thread, reg->successors);
+ __kmp_fast_free(thread, reg);
+ *regp = chain_next;
+ } else {
+ regp = ®->alloc_chain;
+ }
+ }
+ // Free recycled dep list. We could pass this along to the next invocation
+ // of this function instead, but we *mostly* don't do that yet
+ // (ownership/thread safety needs careful consideration if we do that).
+ // The "mostly" concerns irreducible-region edges: we recycle those
+ // opportunistically in __kmp_free_taskgraph.
+ for (kmp_taskgraph_region_dep_t *dep = taskgraph->recycled_deps; dep;) {
+ kmp_taskgraph_region_dep_t *next = dep->next;
+ TGDBG("free dep from recycled list\n");
+ __kmp_fast_free(thread, dep);
+ dep = next;
+ }
+ taskgraph->recycled_deps = nullptr;
+
+ if (__kmp_taskgraph_trace()) {
+ fprintf(stderr, "Processed taskgraph %p (graph_id %" PRIx64 "):\n",
+ taskgraph, taskgraph->graph_id);
+ __kmp_dump_taskgraph_regions(stderr, root_region);
+ }
+
+#ifdef DEBUG_TASKGRAPH
+//__kmp_dump_taskgraph_regions(stderr, root_region);
+//__kmp_dump_raw_taskgraph_regions(stderr, thread, taskgraph,
+// &initial_regions[0], numregions);
+#endif
+
+ KMP_ATOMIC_ST_REL(&taskgraph->status, KMP_TDG_READY);
+
+ return 0;
+}
+#endif
+
+#define NO_DEP_BARRIER (false)
+#define DEP_BARRIER (true)
+
+// returns true if the task has any outstanding dependence
+static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
+ kmp_task_t *task, kmp_dephash_t **hash,
+ bool dep_barrier, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list,
+ kmp_int32 ndeps_noalias,
+ kmp_depend_info_t *noalias_dep_list) {
+ int n_mtxs = 0, dep_all = 0;
+#if KMP_DEBUG
+ kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+#endif
+ KA_TRACE(20, ("__kmp_check_deps: T#%d checking dependences for task %p : %d "
+ "possibly aliased dependences, %d non-aliased dependences : "
+ "dep_barrier=%d .\n",
+ gtid, taskdata, ndeps, ndeps_noalias, dep_barrier));
+
+ dep_all = __kmp_filter_aliased_deps(ndeps, dep_list, task, &n_mtxs);
+
// doesn't need to be atomic as no other thread is going to be accessing this
// node just yet.
// npredecessors is set -1 to ensure that none of the releasing tasks queues
@@ -555,14 +3352,17 @@ static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
// used to pack all npredecessors additions into a single atomic operation at
// the end
int npredecessors;
+ kmp_int32 next_mutex = 0;
if (!dep_all) { // regular dependences
- npredecessors = __kmp_process_deps<true>(gtid, node, hash, dep_barrier,
- ndeps, dep_list, task);
- npredecessors += __kmp_process_deps<false>(
- gtid, node, hash, dep_barrier, ndeps_noalias, noalias_dep_list, task);
+ npredecessors = __kmp_process_deps<normal_deps>(
+ gtid, node, hash, dep_barrier, ndeps, dep_list, task, next_mutex);
+ npredecessors += __kmp_process_deps<normal_deps>(
+ gtid, node, hash, dep_barrier, ndeps_noalias, noalias_dep_list, task,
+ next_mutex, false);
} else { // omp_all_memory dependence
- npredecessors = __kmp_process_dep_all(gtid, node, *hash, dep_barrier, task);
+ npredecessors = __kmp_process_dep_all<normal_deps>(gtid, node, *hash,
+ dep_barrier, task);
}
node->dn.task = task;
diff --git a/openmp/runtime/src/kmp_taskdeps.h b/openmp/runtime/src/kmp_taskdeps.h
index 71e8e69d44593e..e7f23745967e08 100644
--- a/openmp/runtime/src/kmp_taskdeps.h
+++ b/openmp/runtime/src/kmp_taskdeps.h
@@ -40,6 +40,7 @@ static inline void __kmp_node_deref(kmp_info_t *thread, kmp_depnode_t *node) {
}
}
+template <bool refcounting>
static inline void __kmp_depnode_list_free(kmp_info_t *thread,
kmp_depnode_list *list) {
kmp_depnode_list *next;
@@ -47,7 +48,8 @@ static inline void __kmp_depnode_list_free(kmp_info_t *thread,
for (; list; list = next) {
next = list->next;
- __kmp_node_deref(thread, list->node);
+ if (refcounting)
+ __kmp_node_deref(thread, list->node);
#if USE_FAST_MEMORY
__kmp_fast_free(thread, list);
#else
@@ -56,6 +58,7 @@ static inline void __kmp_depnode_list_free(kmp_info_t *thread,
}
}
+template <bool refcounting>
static inline void __kmp_dephash_free_entries(kmp_info_t *thread,
kmp_dephash_t *h) {
for (size_t i = 0; i < h->size; i++) {
@@ -63,12 +66,14 @@ static inline void __kmp_dephash_free_entries(kmp_info_t *thread,
kmp_dephash_entry_t *next;
for (kmp_dephash_entry_t *entry = h->buckets[i]; entry; entry = next) {
next = entry->next_in_bucket;
- __kmp_depnode_list_free(thread, entry->last_set);
- __kmp_depnode_list_free(thread, entry->prev_set);
- __kmp_node_deref(thread, entry->last_out);
- if (entry->mtx_lock) {
- __kmp_destroy_lock(entry->mtx_lock);
- __kmp_free(entry->mtx_lock);
+ __kmp_depnode_list_free<refcounting>(thread, entry->last_set);
+ __kmp_depnode_list_free<refcounting>(thread, entry->prev_set);
+ if (refcounting) {
+ __kmp_node_deref(thread, entry->last_out);
+ if (entry->mtx_lock) {
+ __kmp_destroy_lock(entry->mtx_lock);
+ __kmp_free(entry->mtx_lock);
+ }
}
#if USE_FAST_MEMORY
__kmp_fast_free(thread, entry);
@@ -79,12 +84,14 @@ static inline void __kmp_dephash_free_entries(kmp_info_t *thread,
h->buckets[i] = 0;
}
}
- __kmp_node_deref(thread, h->last_all);
+ if (refcounting)
+ __kmp_node_deref(thread, h->last_all);
h->last_all = NULL;
}
+template <bool refcounting>
static inline void __kmp_dephash_free(kmp_info_t *thread, kmp_dephash_t *h) {
- __kmp_dephash_free_entries(thread, h);
+ __kmp_dephash_free_entries<refcounting>(thread, h);
#if USE_FAST_MEMORY
__kmp_fast_free(thread, h);
#else
@@ -112,7 +119,7 @@ static inline void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task) {
KA_TRACE(
40, ("__kmp_release_deps: T#%d freeing dependencies hash of task %p.\n",
gtid, task));
- __kmp_dephash_free(thread, task->td_dephash);
+ __kmp_dephash_free<true>(thread, task->td_dephash);
task->td_dephash = NULL;
}
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index eae78848988d23..0c398e0de66688 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -17,6 +17,8 @@
#include "kmp_wait_release.h"
#include "kmp_taskdeps.h"
+#undef DEBUG_TASKGRAPH
+
#if OMPT_SUPPORT
#include "ompt-specific.h"
#endif
@@ -731,7 +733,7 @@ static void __kmp_free_task_and_ancestors(kmp_int32 gtid,
"dephash of implicit task %p\n",
gtid, taskdata));
// cleanup dephash of finished implicit task
- __kmp_dephash_free_entries(thread, taskdata->td_dephash);
+ __kmp_dephash_free_entries<true>(thread, taskdata->td_dephash);
}
}
}
@@ -766,6 +768,12 @@ static bool __kmp_track_children_task(kmp_taskdata_t *taskdata) {
return ret;
}
+#if OMP_TASKGRAPH_EXPERIMENTAL
+static bool
+__kmp_taskgraph_exec_descr_finish(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_taskgraph_exec_descr_t *descr);
+#endif
+
// __kmp_task_finish: bookkeeping to do when a task finishes execution
//
// gtid: global thread ID for calling thread
@@ -782,6 +790,9 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
kmp_info_t *thread = __kmp_threads[gtid];
kmp_task_team_t *task_team =
thread->th.th_task_team; // might be NULL for serial teams...
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ bool is_taskgraph;
+#endif
#if KMP_DEBUG
kmp_int32 children = 0;
#endif
@@ -791,6 +802,10 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ is_taskgraph = taskdata->owning_taskgraph;
+#endif
+
if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
// untied task needs to check the counter so that the task structure is not
// freed prematurely
@@ -901,6 +916,19 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
if (ompt)
__ompt_task_finish(task, resumed_task, ompt_task_complete);
#endif
+
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (is_taskgraph) {
+ __kmp_taskgraph_exec_descr_finish(gtid, thread, taskdata->exec_descr);
+ KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks);
+ if (taskdata->td_taskgroup)
+ KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
+ thread->th.th_current_task = resumed_task;
+ resumed_task->td_flags.executing = 1; // resume previous task
+ return;
+ }
+#endif
+
// TODO: What would be the balance between the conditions in the function
// and an atomic operation?
if (__kmp_track_children_task(taskdata)) {
@@ -1119,7 +1147,7 @@ void __kmp_finish_implicit_task(kmp_info_t *thread) {
KA_TRACE(100, ("__kmp_finish_implicit_task: T#%d cleans "
"dephash of implicit task %p\n",
thread->th.th_info.ds.ds_gtid, task));
- __kmp_dephash_free_entries(thread, task->td_dephash);
+ __kmp_dephash_free_entries<true>(thread, task->td_dephash);
}
}
}
@@ -1132,7 +1160,7 @@ void __kmp_finish_implicit_task(kmp_info_t *thread) {
void __kmp_free_implicit_task(kmp_info_t *thread) {
kmp_taskdata_t *task = thread->th.th_current_task;
if (task && task->td_dephash) {
- __kmp_dephash_free(thread, task->td_dephash);
+ __kmp_dephash_free<true>(thread, task->td_dephash);
task->td_dephash = NULL;
}
}
@@ -1147,7 +1175,7 @@ static size_t __kmp_round_up_to_val(size_t size, size_t val) {
}
}
return size;
-} // __kmp_round_up_to_va
+} // __kmp_round_up_to_val
// __kmp_task_alloc: Allocate the taskdata and task data structures for a task
//
@@ -1336,6 +1364,10 @@ kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
taskdata->td_flags.executing = 0;
taskdata->td_flags.complete = 0;
taskdata->td_flags.freed = 0;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ taskdata->owning_taskgraph = nullptr;
+ taskdata->exec_descr = nullptr;
+#endif
KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
// start at one because counts current task and children
KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
@@ -2090,6 +2122,629 @@ kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
return TASK_CURRENT_NOT_QUEUED;
}
+#if OMP_TASKGRAPH_EXPERIMENTAL
+
+// Return the members of the irreducible REGION in topological order, i.e. with
+// each member preceded by its intra-region predecessors. The caller frees the
+// returned array.
+static kmp_taskgraph_region_t **
+__kmp_taskgraph_irreducible_order(kmp_info_t *thread,
+ kmp_taskgraph_region_t *region) {
+ kmp_int32 num_children = region->inner.num_children;
+ kmp_taskgraph_region_t **order =
+ (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+ thread, num_children * sizeof(kmp_taskgraph_region_t *));
+ kmp_int32 outidx = 0;
+
+ for (kmp_int32 c = 0; c < num_children; c++)
+ region->inner.children[c]->mark = TASKGRAPH_UNMARKED;
+
+ for (kmp_int32 c = 0; c < num_children; c++)
+ if (region->inner.children[c]->mark == TASKGRAPH_UNMARKED)
+ __kmp_taskgraph_topological_order(region->inner.children[c], order,
+ &outidx);
+
+ // Members only depend on other members, so the walk cannot leave the region.
+ assert(outidx == num_children);
+ return order;
+}
+
+// The number of exit descrs REGION resolves to, i.e. the length of the pred
+// list __kmp_build_exec_descrs_1 returns for it. A dependency on REGION has to
+// name every one of them.
+static kmp_int32 __kmp_count_region_exits(kmp_taskgraph_region_t *region) {
+ switch (region->type) {
+ case TASKGRAPH_REGION_NODE:
+ case TASKGRAPH_REGION_WAIT:
+ case TASKGRAPH_REGION_ENTRY:
+ case TASKGRAPH_REGION_EXIT:
+ return 1;
+ case TASKGRAPH_REGION_PARALLEL: {
+ // A region with reductions is left through its join descr.
+ if (region->reduce_input)
+ return 1;
+ // All children run concurrently, so all of them are exits.
+ kmp_int32 exits = 0;
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++)
+ exits += __kmp_count_region_exits(region->inner.children[c]);
+ return exits;
+ }
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ // The children are chained, so only the last one is an exit.
+ return __kmp_count_region_exits(
+ region->inner.children[region->inner.num_children - 1]);
+ case TASKGRAPH_REGION_IRREDUCIBLE: {
+ // The members with no successors are the exits.
+ kmp_int32 exits = 0;
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++)
+ if (!region->inner.children[c]->successors)
+ exits += __kmp_count_region_exits(region->inner.children[c]);
+ return exits;
+ }
+ }
+}
+
+// Count the number of exec_descr and successor-list structures that we need
+// to flatten REGION. Follows the structure of __kmp_build_exec_descrs_1.
+
+kmp_int32 __kmp_count_exec_descrs_1(kmp_taskgraph_region_t *region,
+ kmp_int32 npreds, kmp_int32 &successors,
+ kmp_int32 &descrs) {
+ switch (region->type) {
+ case TASKGRAPH_REGION_NODE:
+ case TASKGRAPH_REGION_WAIT:
+ case TASKGRAPH_REGION_ENTRY:
+ case TASKGRAPH_REGION_EXIT: {
+ descrs++;
+ if (npreds > 0) {
+ successors += npreds;
+ }
+ return 1;
+ }
+ case TASKGRAPH_REGION_PARALLEL: {
+ if (npreds > 0)
+ successors += npreds;
+ descrs++;
+ npreds = 1;
+ kmp_int32 out_preds = 0;
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ kmp_int32 child_preds = __kmp_count_exec_descrs_1(
+ region->inner.children[c], npreds, successors, descrs);
+ out_preds += child_preds;
+ }
+ if (region->reduce_input) {
+ // The join descr, which the members' exits do not name.
+ descrs++;
+ return 1;
+ }
+ return out_preds;
+ }
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_EXCLUSIVE: {
+ // In this loop, PREDS can expire per-iteration and be reinitialised. So
+ // we shouldn't alter its refcount.
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ npreds = __kmp_count_exec_descrs_1(region->inner.children[c], npreds,
+ successors, descrs);
+ }
+ return npreds;
+ }
+ case TASKGRAPH_REGION_IRREDUCIBLE: {
+ if (npreds > 0)
+ successors += npreds;
+ descrs++;
+ npreds = 1;
+ kmp_int32 out_preds = 0;
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ kmp_taskgraph_region_t *child_region = region->inner.children[c];
+ kmp_int32 inner_or_outer_preds = npreds;
+ if (child_region->predecessors) {
+ // Mirror the builder: an edge to a member contributes that member's
+ // whole exit set, not a single entry.
+ inner_or_outer_preds = 0;
+ for (kmp_taskgraph_region_dep_t *p = child_region->predecessors; p;
+ p = p->next)
+ inner_or_outer_preds += __kmp_count_region_exits(p->region);
+ }
+ kmp_int32 child_preds = __kmp_count_exec_descrs_1(
+ child_region, inner_or_outer_preds, successors, descrs);
+ if (!child_region->successors)
+ out_preds += child_preds;
+ }
+ return out_preds;
+ }
+ }
+}
+
+void __kmp_count_exec_descrs(kmp_taskgraph_region_t *region,
+ kmp_int32 &successors, kmp_int32 &descrs) {
+ [[maybe_unused]] kmp_int32 npreds;
+ npreds = __kmp_count_exec_descrs_1(region, 0, successors, descrs);
+ // We want to reach the exit...
+ assert(npreds > 0);
+}
+
+kmp_taskgraph_exec_descr_list_t *
+__kmp_exec_descr_list_add(kmp_info_t *thread,
+ kmp_taskgraph_exec_descr_elem_t **recycled_elems,
+ kmp_taskgraph_exec_descr_t *exec_descr,
+ kmp_taskgraph_exec_descr_list_t *list) {
+ kmp_taskgraph_exec_descr_elem_t *head;
+ if (*recycled_elems) {
+ head = *recycled_elems;
+ *recycled_elems = (*recycled_elems)->next;
+ } else
+ head = (kmp_taskgraph_exec_descr_elem_t *)__kmp_fast_allocate(
+ thread, sizeof(kmp_taskgraph_exec_descr_elem_t));
+ head->exec_descr = exec_descr;
+ if (!list) {
+ list = (kmp_taskgraph_exec_descr_list_t *)__kmp_fast_allocate(
+ thread, sizeof(kmp_taskgraph_exec_descr_list_t));
+ list->head = head;
+ list->refcount = 1;
+ list->length = 1;
+ head->next = head;
+ } else {
+ kmp_taskgraph_exec_descr_elem_t *list_head = list->head;
+ kmp_taskgraph_exec_descr_elem_t *list_head_next = list_head->next;
+ list_head->next = head;
+ head->next = list_head_next;
+ list->head = head;
+ list->length++;
+ }
+ return list;
+}
+
+kmp_taskgraph_exec_descr_list_t *__kmp_exec_descr_singleton_list(
+ kmp_info_t *thread, kmp_taskgraph_exec_descr_elem_t **recycled_elems,
+ kmp_taskgraph_exec_descr_t *exec_descr) {
+ return __kmp_exec_descr_list_add(thread, recycled_elems, exec_descr, nullptr);
+}
+
+static kmp_int32
+__kmp_exec_descr_list_len(kmp_taskgraph_exec_descr_list_t *list) {
+ return list ? list->length : 0;
+}
+
+void __kmp_exec_descr_free_recycled(kmp_info_t *thread,
+ kmp_taskgraph_exec_descr_elem_t *elems) {
+ while (elems) {
+ kmp_taskgraph_exec_descr_elem_t *next = elems->next;
+ __kmp_fast_free(thread, elems);
+ elems = next;
+ }
+}
+
+static void
+__kmp_exec_descr_elem_recycle(kmp_taskgraph_exec_descr_elem_t **recycled,
+ kmp_taskgraph_exec_descr_elem_t *elem) {
+ elem->next = *recycled;
+ *recycled = elem;
+}
+
+void __kmp_exec_descr_list_recycle(kmp_info_t *thread,
+ kmp_taskgraph_exec_descr_elem_t **recycled,
+ kmp_taskgraph_exec_descr_list_t *list) {
+ if (!list)
+ return;
+ kmp_taskgraph_exec_descr_elem_t *head = list->head, *walk;
+ walk = head;
+ do {
+ kmp_taskgraph_exec_descr_elem_t *next = walk->next;
+ __kmp_exec_descr_elem_recycle(recycled, walk);
+ walk = next;
+ } while (walk != head);
+ __kmp_fast_free(thread, list);
+}
+
+kmp_taskgraph_exec_descr_list_t *
+__kmp_exec_descr_list_ref(kmp_taskgraph_exec_descr_list_t *list) {
+ if (list)
+ list->refcount++;
+ return list;
+}
+
+kmp_taskgraph_exec_descr_list_t *
+__kmp_exec_descr_list_deref(kmp_info_t *thread,
+ kmp_taskgraph_exec_descr_elem_t **recycled,
+ kmp_taskgraph_exec_descr_list_t *list) {
+ if (!list)
+ return nullptr;
+
+ list->refcount--;
+ if (list->refcount == 0) {
+ __kmp_exec_descr_list_recycle(thread, recycled, list);
+ return nullptr;
+ }
+ return list;
+}
+
+// Destructively append the elements of TO_ADD to DST_LIST. The TO_ADD list
+// header is freed and shouldn't be used after calling this function.
+
+kmp_taskgraph_exec_descr_list_t *
+__kmp_exec_descr_splice_list(kmp_info_t *thread,
+ kmp_taskgraph_exec_descr_list_t *dst_list,
+ kmp_taskgraph_exec_descr_list_t *to_add) {
+ if (dst_list == nullptr && to_add == nullptr)
+ return nullptr;
+ else if (to_add == nullptr)
+ return dst_list;
+ else if (dst_list == nullptr) {
+ // We're splicing into a "new" list, so reset the refcount.
+ to_add->refcount = 1;
+ return to_add;
+ }
+
+ kmp_taskgraph_exec_descr_elem_t *dst_head = dst_list->head;
+ kmp_taskgraph_exec_descr_elem_t *dst_head_next = dst_head->next;
+ kmp_taskgraph_exec_descr_elem_t *add_head = to_add->head;
+ kmp_taskgraph_exec_descr_elem_t *add_head_next = add_head->next;
+
+ dst_list->length += to_add->length;
+ dst_head->next = add_head_next;
+ add_head->next = dst_head_next;
+
+ // Order doesn't really matter, but this makes the head pointer point to the
+ // start of the added list, so TO_ADD is neatly prepended to the dst list,
+ // rather than having (old head->appended list->remainder of old list).
+ dst_list->head = add_head;
+
+ // Free the added list header.
+ __kmp_fast_free(thread, to_add);
+
+ return dst_list;
+}
+
+// The PREDS list gets consumed ONCE by a call to this function, so if you need
+// to keep it alive, take another reference on it.
+
+kmp_taskgraph_exec_descr_list_t *__kmp_build_exec_descrs_1(
+ kmp_info_t *thread, kmp_taskgraph_exec_descr_t *exec_descr_arr,
+ kmp_int32 &idx, kmp_taskgraph_exec_descr_elem_t **recycled,
+ kmp_taskgraph_region_t *region, kmp_taskgraph_exec_descr_list_t *preds,
+ kmp_taskgraph_exec_descr_elem_t **successor_tail) {
+ auto descr_for_region =
+ [&](kmp_taskgraph_region_t *region,
+ kmp_int32 npreds) -> kmp_taskgraph_exec_descr_t * {
+ kmp_taskgraph_exec_descr_t *ed = nullptr;
+ if (region->exec_descr) {
+ assert(region->exec_descr->region == region);
+ ed = region->exec_descr;
+ } else {
+ assert(npreds >= 0);
+ ed = &exec_descr_arr[idx++];
+ ed->region = region;
+ ed->indegree = npreds;
+ ed->successors = nullptr;
+ ed->join = nullptr;
+ region->exec_descr = ed;
+ }
+ return ed;
+ };
+ auto fill = [&](kmp_taskgraph_exec_descr_t *exec_descr,
+ kmp_taskgraph_exec_descr_list_t *preds_list) -> void {
+ if (!preds_list)
+ return;
+ kmp_taskgraph_exec_descr_elem_t *pred = preds_list->head, *walk;
+ walk = pred;
+ do {
+ kmp_taskgraph_exec_descr_t *pred_descr = walk->exec_descr;
+ // Build a simple linked list (null-terminated) of successors. We're
+ // using a bump-allocated region because we know how many successors
+ // we need (we counted already).
+ kmp_taskgraph_exec_descr_elem_t *new_successor = *successor_tail;
+ (*successor_tail)++;
+ new_successor->exec_descr = exec_descr;
+ new_successor->next = pred_descr->successors;
+ pred_descr->successors = new_successor;
+ walk = walk->next;
+ } while (walk != pred);
+ };
+ switch (region->type) {
+ case TASKGRAPH_REGION_NODE:
+ case TASKGRAPH_REGION_WAIT:
+ case TASKGRAPH_REGION_ENTRY:
+ case TASKGRAPH_REGION_EXIT: {
+ kmp_taskgraph_exec_descr_t *exec_descr =
+ descr_for_region(region, __kmp_exec_descr_list_len(preds));
+ // Edit the taskdata for this specific instantiation. At present the
+ // task/taskdata structures cannot be used simultaneously by different
+ // threads. We could duplicate the structures to allow simultaneous issue,
+ // but that's not done yet. The exec_descr can already by thread-local,
+ // in principle, but for now it points to the taskgraph's single copy
+ // of each task/taskdata structure.
+ if (region->type == TASKGRAPH_REGION_NODE) {
+ kmp_task_t *task = exec_descr->region->task.node->task;
+ kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+ taskdata->exec_descr = exec_descr;
+ }
+ fill(exec_descr, preds);
+ __kmp_exec_descr_list_deref(thread, recycled, preds);
+ return __kmp_exec_descr_singleton_list(thread, recycled, exec_descr);
+ }
+ case TASKGRAPH_REGION_PARALLEL: {
+ kmp_int32 npreds = __kmp_exec_descr_list_len(preds);
+ kmp_taskgraph_exec_descr_t *gather_descr = descr_for_region(region, npreds);
+ fill(gather_descr, preds);
+ __kmp_exec_descr_list_deref(thread, recycled, preds);
+ preds = __kmp_exec_descr_singleton_list(thread, recycled, gather_descr);
+ kmp_taskgraph_exec_descr_list_t *out_preds = nullptr;
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ kmp_taskgraph_exec_descr_list_t *child_preds;
+ // The preds are reused for each iteration, so keep them alive over this
+ // iteration of the loop.
+ __kmp_exec_descr_list_ref(preds);
+ child_preds = __kmp_build_exec_descrs_1(
+ thread, exec_descr_arr, idx, recycled, region->inner.children[c],
+ preds, successor_tail);
+ out_preds = __kmp_exec_descr_splice_list(thread, out_preds, child_preds);
+ }
+ __kmp_exec_descr_list_deref(thread, recycled, preds);
+ if (region->reduce_input) {
+ // Whatever follows the region must see the combined reductions, so it
+ // waits on the join descr, which the region's descr releases once its
+ // taskgroup has ended, instead of on the members themselves.
+ __kmp_exec_descr_list_deref(thread, recycled, out_preds);
+ kmp_taskgraph_exec_descr_t *join = &exec_descr_arr[idx++];
+ join->region = region;
+ join->indegree = 0;
+ join->successors = nullptr;
+ join->join = nullptr;
+ gather_descr->join = join;
+ return __kmp_exec_descr_singleton_list(thread, recycled, join);
+ }
+ return out_preds;
+ }
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_EXCLUSIVE: {
+ // In this loop, PREDS expires per-iteration and a fresh list is returned
+ // (with refcount==1). So we don't need to make any further refcount
+ // adjustments.
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ preds = __kmp_build_exec_descrs_1(thread, exec_descr_arr, idx, recycled,
+ region->inner.children[c], preds,
+ successor_tail);
+ }
+ return preds;
+ }
+ case TASKGRAPH_REGION_IRREDUCIBLE: {
+ kmp_int32 npreds = __kmp_exec_descr_list_len(preds);
+ kmp_taskgraph_exec_descr_t *gather_descr = descr_for_region(region, npreds);
+ fill(gather_descr, preds);
+ __kmp_exec_descr_list_deref(thread, recycled, preds);
+ preds = __kmp_exec_descr_singleton_list(thread, recycled, gather_descr);
+
+ // A member's dependencies name sibling members, and we resolve each one to
+ // the exit descrs recorded on it (see EXEC_DESCRS_OUT below). Members are
+ // stored in reverse preorder, so walk them in topological order instead:
+ // that way a predecessor has always been built by the time we need it.
+ kmp_int32 num_children = region->inner.num_children;
+ kmp_taskgraph_region_t **order =
+ __kmp_taskgraph_irreducible_order(thread, region);
+
+ kmp_taskgraph_exec_descr_list_t *out_preds = nullptr;
+ for (kmp_int32 c = 0; c < num_children; c++) {
+ kmp_taskgraph_region_t *child_region = order[c];
+ kmp_taskgraph_exec_descr_list_t *inner_or_outer_preds = preds;
+ if (child_region->predecessors) {
+ inner_or_outer_preds = nullptr;
+ for (kmp_taskgraph_region_dep_t *p = child_region->predecessors; p;
+ p = p->next) {
+ assert(p->region);
+ // An edge to a compound member means "after everything in it", so
+ // depend on that member's whole exit set rather than on its entry
+ // descr. For a leaf member the exit set is just its own descr.
+ assert(p->region->exec_descrs_out);
+ kmp_taskgraph_exec_descr_elem_t *head =
+ p->region->exec_descrs_out->head;
+ kmp_taskgraph_exec_descr_elem_t *walk = head;
+ do {
+ inner_or_outer_preds = __kmp_exec_descr_list_add(
+ thread, recycled, walk->exec_descr, inner_or_outer_preds);
+ walk = walk->next;
+ } while (walk != head);
+ }
+ } else {
+ // PREDS may be reused on successive iterations of the loop: make sure
+ // it stays alive.
+ __kmp_exec_descr_list_ref(preds);
+ }
+ // If INNER_OR_OUTER_PREDS is local, it will be consumed by the call. If
+ // it is the incoming PREDS, we bumped the reference count above. In
+ // either case the recursive call consumes it once.
+ kmp_taskgraph_exec_descr_list_t *child_preds = __kmp_build_exec_descrs_1(
+ thread, exec_descr_arr, idx, recycled, child_region,
+ inner_or_outer_preds, successor_tail);
+ if (!child_region->successors) {
+ // A region exit. Having no successors, this child is named by no
+ // sibling, so nothing reads its exit set back and the destructive
+ // splice is safe.
+ out_preds =
+ __kmp_exec_descr_splice_list(thread, out_preds, child_preds);
+ } else {
+ // Later siblings resolve their edges to this child through
+ // EXEC_DESCRS_OUT, so keep a reference for the rest of the loop.
+ child_region->exec_descrs_out = child_preds;
+ }
+ }
+ // Release the exit sets: nothing outside this loop reads them, and leaving
+ // the pointers behind would leave dangling values in a structure that
+ // outlives the build.
+ for (kmp_int32 c = 0; c < num_children; c++) {
+ if (order[c]->exec_descrs_out) {
+ __kmp_exec_descr_list_deref(thread, recycled,
+ order[c]->exec_descrs_out);
+ order[c]->exec_descrs_out = nullptr;
+ }
+ }
+ __kmp_fast_free(thread, order);
+ // We do this unconditionally even though it looks like it could depend on
+ // the flow in the loop above, but the (explicit) ref by the _list_ref call
+ // and (implicit) deref by __kmp_build_exec_descrs balance out, so it's OK.
+ // The next line dereferences PREDS for *this* function activation, i.e.
+ // for our caller.
+ __kmp_exec_descr_list_deref(thread, recycled, preds);
+ return out_preds;
+ }
+ }
+}
+
+kmp_taskgraph_exec_descr_t *
+__kmp_build_exec_descrs(kmp_info_t *thread, kmp_taskgraph_region_t *region,
+ kmp_int32 *num_descrs) {
+ kmp_int32 ndescrs = 0, nsuccessors = 0;
+ __kmp_count_exec_descrs(region, nsuccessors, ndescrs);
+ char *arena = (char *)__kmp_thread_malloc(
+ thread, ndescrs * sizeof(kmp_taskgraph_exec_descr_t) +
+ nsuccessors * sizeof(kmp_taskgraph_exec_descr_elem_t));
+ kmp_taskgraph_exec_descr_t *exec_descr_arr =
+ (kmp_taskgraph_exec_descr_t *)arena;
+ kmp_taskgraph_exec_descr_elem_t *successors =
+ (kmp_taskgraph_exec_descr_elem_t *)&exec_descr_arr[ndescrs];
+ kmp_taskgraph_exec_descr_elem_t *successor_tail = successors;
+ kmp_taskgraph_exec_descr_elem_t *recycled_elems = nullptr;
+ kmp_int32 idx = 0;
+ kmp_taskgraph_exec_descr_list_t *out_preds =
+ __kmp_build_exec_descrs_1(thread, exec_descr_arr, idx, &recycled_elems,
+ region, nullptr, &successor_tail);
+ assert(idx == ndescrs);
+ assert(successor_tail - successors == nsuccessors);
+ __kmp_exec_descr_list_deref(thread, &recycled_elems, out_preds);
+ __kmp_exec_descr_free_recycled(thread, recycled_elems);
+ *num_descrs = ndescrs;
+ return exec_descr_arr;
+}
+
+/// Reset, reparent and regroup the recorded task TASK and re-invoke it.
+
+static void __kmp_omp_tg_task(kmp_int32 gtid, kmp_task_t *task,
+ kmp_taskgroup_t *taskgroup,
+ kmp_taskdata_t *parent_taskdata,
+ bool serialize_immediate) {
+ kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+ taskdata->td_parent = parent_taskdata;
+
+ // The team is the replaying one, not the recording one: a graph recorded in
+ // a serialized team and replayed in a real one would otherwise run its tasks
+ // as serialized ones there, skipping dependence tracking for the tasks they
+ // create and returning memory to the wrong thread's pool.
+ kmp_team_t *team = __kmp_threads[gtid]->th.th_team;
+ taskdata->td_team = team;
+ taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
+ taskdata->td_flags.team_serial = team->t.t_serialized ? 1 : 0;
+
+ taskdata->td_flags.complete = 0;
+ taskdata->td_flags.started = 0;
+ taskdata->td_flags.freed = 0;
+ taskdata->td_flags.executing = 0;
+ taskdata->td_flags.task_serial =
+ (parent_taskdata->td_flags.final || taskdata->td_flags.team_serial ||
+ taskdata->td_flags.tasking_ser);
+
+ // An untied task left its last part's id behind; it starts again at 0.
+ task->part_id = 0;
+ KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0);
+ KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
+ // Start at one because counter represents current task and children.
+ KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
+
+ taskdata->td_taskgroup = taskgroup;
+ KMP_ATOMIC_INC(&taskgroup->count);
+ KMP_ATOMIC_INC(&parent_taskdata->td_incomplete_child_tasks);
+ if (parent_taskdata->td_flags.tasktype == TASK_EXPLICIT)
+ KMP_ATOMIC_INC(&parent_taskdata->td_allocated_child_tasks);
+
+ __kmp_omp_task(gtid, task, false);
+}
+
+struct kmp_taskred_input;
+template <typename T>
+void *__kmp_task_reduction_init(int gtid, int num, T *data);
+
+// Decrement DESCR's predecessor count; if it reaches zero, fire it. TASKGROUP
+// is the taskgroup any task spawned by DESCR should join.
+static void __kmp_taskgraph_exec_descr_start(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_taskgraph_exec_descr_t *descr,
+ kmp_taskgroup_t *taskgroup) {
+ kmp_int32 npredecessors = KMP_ATOMIC_DEC(&descr->npredecessors) - 1;
+ if (npredecessors > 0)
+ return;
+
+ switch (descr->region->type) {
+ case TASKGRAPH_REGION_NODE: {
+ // Spawn the task; its successors are triggered later, on completion, by
+ // __kmp_taskgraph_exec_descr_finish.
+ kmp_taskgraph_node_t *node = descr->region->task.node;
+ kmp_taskdata_t *current_taskdata = thread->th.th_current_task;
+ if (descr->region->mutexset)
+ __kmp_taskgraph_acquire_locks(gtid, descr->region->owner, descr->region);
+ __kmp_omp_tg_task(gtid, node->task, taskgroup, current_taskdata, false);
+ break;
+ }
+ case TASKGRAPH_REGION_PARALLEL:
+ if (descr->region->reduce_input) {
+ // If there are reductions associated with this parallel region, its
+ // members run in a taskgroup of their own, whose end combines the
+ // reductions into their original variables.
+ __kmpc_taskgroup(/*loc=*/nullptr, gtid);
+ kmp_taskgroup_t *members_taskgroup =
+ thread->th.th_current_task->td_taskgroup;
+ __kmp_task_reduction_init(
+ gtid, descr->region->reduce_input->reduce_num_data,
+ (struct kmp_taskred_input *)descr->region->reduce_input->reduce_data);
+ for (kmp_taskgraph_exec_descr_elem_t *s = descr->successors; s;
+ s = s->next)
+ __kmp_taskgraph_exec_descr_start(gtid, thread, s->exec_descr,
+ members_taskgroup);
+ __kmpc_end_taskgroup(/*loc=*/nullptr, gtid);
+ for (kmp_taskgraph_exec_descr_elem_t *s = descr->join->successors; s;
+ s = s->next)
+ __kmp_taskgraph_exec_descr_start(gtid, thread, s->exec_descr,
+ taskgroup);
+ break;
+ }
+ KMP_FALLTHROUGH();
+ case TASKGRAPH_REGION_WAIT:
+ case TASKGRAPH_REGION_IRREDUCIBLE:
+ for (kmp_taskgraph_exec_descr_elem_t *s = descr->successors; s; s = s->next)
+ __kmp_taskgraph_exec_descr_start(gtid, thread, s->exec_descr, taskgroup);
+ break;
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_ENTRY:
+ case TASKGRAPH_REGION_EXIT: {
+ // We don't expect to see any of these region types here.
+ abort();
+ }
+ }
+}
+
+// Called when DESCR's task completes: release its dataflow successors into the
+// same taskgroup the task ran in. Returns true iff there were any successors.
+static bool
+__kmp_taskgraph_exec_descr_finish(kmp_int32 gtid, kmp_info_t *thread,
+ kmp_taskgraph_exec_descr_t *descr) {
+ KMP_DEBUG_ASSERT(descr->region->type == TASKGRAPH_REGION_NODE);
+ if (descr->region->mutexset)
+ __kmp_taskgraph_release_locks(gtid, descr->region->owner, descr->region);
+ kmp_task_t *task = descr->region->task.node->task;
+ kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+ taskdata->td_flags.started = 0;
+ taskdata->td_flags.executing = 0;
+ taskdata->td_flags.complete = 0;
+ taskdata->td_flags.freed = 0;
+ bool any_successors = false;
+ for (kmp_taskgraph_exec_descr_elem_t *s = descr->successors; s; s = s->next) {
+ any_successors = true;
+ __kmp_taskgraph_exec_descr_start(gtid, thread, s->exec_descr,
+ taskdata->td_taskgroup);
+ }
+ return any_successors;
+}
+#endif
+
// Task Reduction implementation
//
// Note: initial implementation didn't take into account the possibility
@@ -2277,6 +2932,57 @@ void *__kmpc_taskred_init(int gtid, int num, void *data) {
return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data);
}
+#if OMP_TASKGRAPH_EXPERIMENTAL
+// The taskgraph record that a construct TASK encounters is recorded into, if
+// any: that of the innermost taskgroup being recorded, if TASK encountered its
+// taskgraph construct. A task generated in the taskgraph region starts in the
+// region's taskgroup, but its region is not part of the taskgraph region
+// (OpenMP 6.0 [2], region) -- a replay runs it again, constructs and all -- so
+// nothing it encounters is recorded, a construct with a 'replayable' clause
+// included. Nor can an outer record be TASK's if the innermost is not, since
+// TASK's own taskgroups are the innermost.
+static kmp_taskgraph_record_t *
+__kmp_taskgraph_task_recording(kmp_taskdata_t *task) {
+ for (kmp_taskgroup_t *taskgroup = task->td_taskgroup; taskgroup;
+ taskgroup = taskgroup->parent) {
+ kmp_taskgraph_record_t *rec =
+ KMP_ATOMIC_LD_ACQ(&taskgroup->taskgraph.recording);
+ if (rec)
+ return taskgroup->taskgraph.recording_task == task ? rec : nullptr;
+ }
+
+ return nullptr;
+}
+
+void *__kmpc_taskgraph_taskred_init(kmp_int32 gtid, kmp_int32 num, void *data) {
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgroup_t *taskgroup = thread->th.th_current_task->td_taskgroup;
+ kmp_taskgraph_record_t *rec =
+ __kmp_taskgraph_task_recording(thread->th.th_current_task);
+
+ if (rec) {
+ kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&rec->status);
+ if (status == KMP_TDG_RECORDING) {
+ kmp_taskgraph_reduce_input_data_t *input_data =
+ (kmp_taskgraph_reduce_input_data_t *)__kmp_fast_allocate(
+ thread, sizeof(kmp_taskgraph_reduce_input_data_t));
+ // The compiler might build the reduction input data on the stack, so
+ // we must make a copy.
+ input_data->reduce_data =
+ __kmp_fast_allocate(thread, sizeof(kmp_taskred_input_t) * num);
+ KMP_MEMCPY(input_data->reduce_data, data,
+ sizeof(kmp_taskred_input_t) * num);
+ ;
+ input_data->reduce_num_data = num;
+ taskgroup->taskgraph.reduce_input = input_data;
+ } else if (status == KMP_TDG_READY)
+ assert(false &&
+ "unexpected __kmpc_taskgraph_taskred_init with ready taskgraph");
+ }
+ return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data);
+}
+#endif
+
// Copy task reduction data (except for shared pointers).
template <typename T>
void __kmp_task_reduction_init_copy(kmp_info_t *thr, int num, T *data,
@@ -2313,7 +3019,11 @@ void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) {
return data; // nothing to do
kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (tg == NULL || thread->th.th_current_task->owning_taskgraph)
+#else
if (tg == NULL)
+#endif
tg = thread->th.th_current_task->td_taskgroup;
KMP_ASSERT(tg != NULL);
kmp_taskred_data_t *arr;
@@ -2524,6 +3234,11 @@ void __kmpc_taskgroup(ident_t *loc, int gtid) {
tg_new->reduce_data = NULL;
tg_new->reduce_num_data = 0;
tg_new->gomp_data = NULL;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ tg_new->taskgraph.recording = nullptr;
+ tg_new->taskgraph.recording_task = nullptr;
+ tg_new->taskgraph.reduce_input = nullptr;
+#endif
taskdata->td_taskgroup = tg_new;
#if OMPT_SUPPORT && OMPT_OPTIONAL
@@ -2679,6 +3394,23 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
__kmp_task_reduction_fini(thread, taskgroup);
}
}
+
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ // The stash is normally adopted by the next node recorded in this taskgroup,
+ // which then owns it. It can legitimately still be here: the construct that
+ // left it may have turned out not to be a replayable construct, in which
+ // case no node was recorded for it and nothing will come back for the stash.
+ // (A `replayable` clause whose argument is a compile-time constant is
+ // resolved by the compiler, which routes such a construct to the ordinary
+ // reduction init instead; this covers the non-constant case.) No node can be
+ // recorded in a taskgroup that is ending, so free it here rather than leak.
+ if (taskgroup->taskgraph.reduce_input) {
+ __kmp_fast_free(thread, taskgroup->taskgraph.reduce_input->reduce_data);
+ __kmp_fast_free(thread, taskgroup->taskgraph.reduce_input);
+ taskgroup->taskgraph.reduce_input = nullptr;
+ }
+#endif
+
// Restore parent taskgroup for the current task
taskdata->td_taskgroup = taskgroup->parent;
__kmp_thread_free(thread, taskgroup);
@@ -2695,6 +3427,48 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
#endif
}
+#if OMP_TASKGRAPH_EXPERIMENTAL
+static void __kmp_replay_taskgraph(kmp_int32 gtid,
+ kmp_taskdata_t *current_taskdata,
+ kmp_taskgraph_record_t *taskgraph,
+ kmp_intptr_t graph_id,
+ kmp_taskgroup_t *taskgroup) {
+ kmp_info_t *thread = __kmp_threads[gtid];
+
+ // An empty graph (no recorded task nodes) has nothing to replay.
+ // Bail out early.
+ if (taskgraph->num_tasks == 0)
+ return;
+
+ // Build the flat dataflow DAG once per taskgraph (cached on the record);
+ // re-arm the predecessor counters on every replay.
+ if (!taskgraph->exec_descrs) {
+ kmp_int32 num_descrs;
+ taskgraph->exec_descrs =
+ __kmp_build_exec_descrs(thread, taskgraph->root, &num_descrs);
+ taskgraph->num_exec_descrs = num_descrs;
+ }
+
+ kmp_taskgraph_exec_descr_t *exec_descrs = taskgraph->exec_descrs;
+ for (kmp_size_t i = 0; i < taskgraph->num_exec_descrs; i++) {
+ assert(exec_descrs[i].indegree >= 0);
+ exec_descrs[i].npredecessors = exec_descrs[i].indegree;
+ }
+
+ // Allocate all the mutexes that tasks (or target regions) need within the
+ // taskgraph.
+ if (taskgraph->num_mutexes > 0 && !taskgraph->node_mutexes) {
+ taskgraph->node_mutexes = (kmp_lock_t *)__kmp_thread_malloc(
+ thread, taskgraph->num_mutexes * sizeof(kmp_lock_t));
+ for (kmp_int32 i = 0; i < taskgraph->num_mutexes; i++)
+ __kmp_init_lock(&taskgraph->node_mutexes[i]);
+ }
+
+ __kmp_taskgraph_exec_descr_start(gtid, thread, taskgraph->exec_descrs,
+ taskgroup);
+}
+#endif
+
static kmp_task_t *__kmp_get_priority_task(kmp_int32 gtid,
kmp_task_team_t *task_team,
kmp_int32 is_constrained) {
@@ -4287,7 +5061,12 @@ void __kmp_fulfill_event(kmp_event_t *event) {
// taskloop_recur: used only when dealing with taskgraph,
// indicating whether we need to update task->td_task_id
// returns: a pointer to the allocated kmp_task_t structure (task).
-kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) {
+kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ ,
+ bool taskgraph
+#endif
+) {
kmp_task_t *task;
kmp_taskdata_t *taskdata;
kmp_taskdata_t *taskdata_src = KMP_TASK_TO_TASKDATA(task_src);
@@ -4333,7 +5112,12 @@ kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) {
// Only need to keep track of child task counts if team parallel and tasking
// not serialized
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser ||
+ taskgraph)) {
+#else
if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) {
+#endif
KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks);
if (parent_task->td_taskgroup)
KMP_ATOMIC_INC(&parent_task->td_taskgroup->count);
@@ -4466,6 +5250,48 @@ class kmp_taskloop_bounds_t {
}
};
+#if OMP_TASKGRAPH_EXPERIMENTAL
+kmp_taskgraph_node_t *
+__kmp_taskgraph_node_alloc(kmp_taskgraph_record_t *rec, kmp_task_t *task,
+ kmp_size_t *index_p = nullptr) {
+ kmp_int32 gtid = rec->gtid;
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgraph_node_t *new_task = nullptr;
+
+ __kmp_acquire_lock(&rec->map_lock, gtid);
+
+ if (!rec->record_map) {
+ rec->nodes_allocated = 4;
+ rec->record_map = (kmp_taskgraph_node_t *)__kmp_thread_malloc(
+ thread, rec->nodes_allocated * sizeof(kmp_taskgraph_node_t));
+ }
+
+ if (rec->num_tasks >= rec->nodes_allocated) {
+ rec->record_map = (kmp_taskgraph_node_t *)__kmp_thread_realloc(
+ thread, rec->record_map,
+ 2 * rec->nodes_allocated * sizeof(kmp_taskgraph_node_t));
+ rec->nodes_allocated *= 2;
+ }
+
+ new_task = &rec->record_map[rec->num_tasks];
+ if (index_p)
+ *index_p = rec->num_tasks;
+ ++rec->num_tasks;
+
+ __kmp_release_lock(&rec->map_lock, gtid);
+
+ new_task->task = task;
+ new_task->taskloop_task = false;
+ new_task->undeferred = false;
+ new_task->reduce_input = nullptr;
+ new_task->ndeps = 0;
+ new_task->dep_list = nullptr;
+ new_task->cfg_successor = -1;
+
+ return new_task;
+}
+#endif
+
// __kmp_taskloop_linear: Start tasks of the taskloop linearly
//
// loc Source location information
@@ -4482,15 +5308,21 @@ class kmp_taskloop_bounds_t {
// tc Iterations count
// task_dup Tasks duplication routine
// codeptr_ra Return address for OMPT events
-void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
- kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
- kmp_uint64 ub_glob, kmp_uint64 num_tasks,
- kmp_uint64 grainsize, kmp_uint64 extras,
- kmp_int64 last_chunk, kmp_uint64 tc,
+static void
+__kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task, kmp_uint64 *lb,
+ kmp_uint64 *ub, kmp_int64 st, kmp_int32 nogroup,
+ kmp_uint64 ub_glob, kmp_uint64 num_tasks,
+ kmp_uint64 grainsize, kmp_uint64 extras,
+ kmp_int64 last_chunk, kmp_uint64 tc,
#if OMPT_SUPPORT
- void *codeptr_ra,
+ void *codeptr_ra,
+#endif
+ void *task_dup
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ ,
+ kmp_taskgraph_record_t *taskgraph_rec = nullptr
#endif
- void *task_dup) {
+) {
KMP_COUNT_BLOCK(OMP_TASKLOOP);
KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling);
p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
@@ -4503,6 +5335,9 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
kmp_taskdata_t *current_task = thread->th.th_current_task;
kmp_task_t *next_task;
kmp_int32 lastpriv = 0;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ kmp_int32 taskloop_prev_idx = -1, taskloop_first_idx = -1;
+#endif
KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
(last_chunk < 0 ? last_chunk : extras));
@@ -4543,7 +5378,11 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
}
}
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ next_task = __kmp_task_dup_alloc(thread, task, /*taskgraph=*/taskgraph_rec);
+#else
next_task = __kmp_task_dup_alloc(thread, task); // allocate new task
+#endif
kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task);
kmp_taskloop_bounds_t next_task_bounds =
@@ -4565,24 +5404,75 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
gtid, i, next_task, lower, upper, st,
next_task_bounds.get_lower_offset(),
next_task_bounds.get_upper_offset()));
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (taskgraph_rec) {
+ kmp_size_t rec_index = -1;
+ // Record the task in the taskgraph.
+ kmp_taskgraph_node_t *node =
+ __kmp_taskgraph_node_alloc(taskgraph_rec, next_task, &rec_index);
+ kmp_taskgroup_t *taskgroup = current_task->td_taskgroup;
+ if (taskgroup->taskgraph.reduce_input) {
+ node->reduce_input = taskgroup->taskgraph.reduce_input;
+ taskgroup->taskgraph.reduce_input = nullptr;
+ }
+ node->taskloop_task = true;
+ next_taskdata->owning_taskgraph = taskgraph_rec;
+ // FIXME: These dependency fields might be back-filled by the as-yet
+ // unimplemented task_iteration subsidiary directive. We'll need a way
+ // to locate the correct task given the value of the iteration variable,
+ // or similar.
+ node->ndeps = 0;
+ node->dep_list = nullptr;
+ if (nogroup)
+ taskgraph_rec->record_map[rec_index].cfg_successor = -1;
+ else if (taskloop_prev_idx != -1)
+ taskgraph_rec->record_map[taskloop_prev_idx].cfg_successor = rec_index;
+ if (taskloop_first_idx == -1)
+ taskloop_first_idx = rec_index;
+ taskloop_prev_idx = rec_index;
+ } else {
+#endif
#if OMPT_SUPPORT
- __kmp_omp_taskloop_task(NULL, gtid, next_task,
- codeptr_ra); // schedule new task
+ __kmp_omp_taskloop_task(NULL, gtid, next_task,
+ codeptr_ra); // schedule new task
#if OMPT_OPTIONAL
- if (ompt_enabled.ompt_callback_dispatch) {
- OMPT_GET_DISPATCH_CHUNK(next_taskdata->ompt_task_info.dispatch_chunk,
- lower, upper, st);
- }
+ if (ompt_enabled.ompt_callback_dispatch) {
+ OMPT_GET_DISPATCH_CHUNK(next_taskdata->ompt_task_info.dispatch_chunk,
+ lower, upper, st);
+ }
#endif // OMPT_OPTIONAL
#else
__kmp_omp_task(gtid, next_task, true); // schedule new task
+#endif
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ }
#endif
lower = upper + st; // adjust lower bound for the next iteration
}
- // free the pattern task and exit
- __kmp_task_start(gtid, task, current_task); // make internal bookkeeping
- // do not execute the pattern task, just do internal bookkeeping
- __kmp_task_finish<false>(gtid, task, current_task);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (taskgraph_rec) {
+ if (taskloop_prev_idx != -1 && !nogroup) {
+ // Create a node to act as an "end group" marker.
+ kmp_size_t endgroup_idx = -1;
+ kmp_taskgraph_node_t *endgrpnode =
+ __kmp_taskgraph_node_alloc(taskgraph_rec, nullptr, &endgroup_idx);
+ endgrpnode->taskloop_task = true;
+ // Point all the cfg_successor indices to this node now.
+ for (kmp_int32 looptask = taskloop_first_idx; looptask != -1;) {
+ kmp_int32 next_task = taskgraph_rec->record_map[looptask].cfg_successor;
+ taskgraph_rec->record_map[looptask].cfg_successor = endgroup_idx;
+ looptask = next_task;
+ }
+ }
+ } else {
+#endif
+ // free the pattern task and exit
+ __kmp_task_start(gtid, task, current_task); // make internal bookkeeping
+ // do not execute the pattern task, just do internal bookkeeping
+ __kmp_task_finish<false>(gtid, task, current_task);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ }
+#endif
}
// Structure to keep taskloop parameters for auxiliary task
@@ -4652,8 +5542,8 @@ int __kmp_taskloop_task(int gtid, void *ptask) {
#endif
task_dup);
else
- __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
- grainsize, extras, last_chunk, tc,
+ __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, /*nogroup=*/true,
+ ub_glob, num_tasks, grainsize, extras, last_chunk, tc,
#if OMPT_SUPPORT
codeptr_ra,
#endif
@@ -4741,7 +5631,11 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
lb1 = ub0 + st;
// create pattern task for 2nd half of the loop
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ next_task = __kmp_task_dup_alloc(thread, task, /*taskgraph=*/false);
+#else
next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task
+#endif
// adjust lower bound (upper bound is not changed) for the 2nd half
*(kmp_uint64 *)((char *)next_task + lower_offset) = lb1;
if (ptask_dup != NULL) // construct firstprivates, etc.
@@ -4774,6 +5668,11 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
p->codeptr_ra = codeptr_ra;
#endif
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ kmp_taskdata_t *new_task_data = KMP_TASK_TO_TASKDATA(new_task);
+ new_task_data->owning_taskgraph = nullptr;
+#endif
+
#if OMPT_SUPPORT
// schedule new task with correct return address for OMPT events
__kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra);
@@ -4790,8 +5689,8 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
#endif
task_dup);
else
- __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0,
- gr_size0, ext0, last_chunk0, tc0,
+ __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, /*nogroup=*/true,
+ ub_glob, n_tsk0, gr_size0, ext0, last_chunk0, tc0,
#if OMPT_SUPPORT
codeptr_ra,
#endif
@@ -4803,14 +5702,22 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
int nogroup, int sched, kmp_uint64 grainsize,
- int modifier, void *task_dup) {
+ int modifier, void *task_dup
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ ,
+ kmp_taskgraph_record_t *taskgraph_rec = nullptr
+#endif
+) {
kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
KMP_DEBUG_ASSERT(task != NULL);
if (nogroup == 0) {
#if OMPT_SUPPORT && OMPT_OPTIONAL
OMPT_STORE_RETURN_ADDRESS(gtid);
#endif
- __kmpc_taskgroup(loc, gtid);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (!taskgraph_rec)
+#endif
+ __kmpc_taskgroup(loc, gtid);
}
// =========================================================================
// calculate loop parameters
@@ -4842,17 +5749,27 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
}
if (tc == 0) {
KA_TRACE(20, ("__kmp_taskloop(exit): T#%d zero-trip loop\n", gtid));
- // free the pattern task and exit
- __kmp_task_start(gtid, task, current_task);
- // do not execute anything for zero-trip loop
- __kmp_task_finish<false>(gtid, task, current_task);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (!taskgraph_rec) {
+#endif
+ // free the pattern task and exit
+ __kmp_task_start(gtid, task, current_task);
+ // do not execute anything for zero-trip loop
+ __kmp_task_finish<false>(gtid, task, current_task);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ }
+#endif
return;
}
#if OMPT_SUPPORT && OMPT_OPTIONAL
ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (ompt_enabled.ompt_callback_work && !taskgraph_rec) {
+#else
if (ompt_enabled.ompt_callback_work) {
+#endif
ompt_callbacks.ompt_callback(ompt_callback_work)(
ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data),
&(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0));
@@ -4909,14 +5826,31 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
KMP_DEBUG_ASSERT(num_tasks > 0);
// =========================================================================
- // check if clause value first
- // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native)
- if (if_val == 0) { // if(0) specified, mark task as serial
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ // Handle taskgraph case first. We just generate tasks and record them in
+ // the graph, but we do not execute them here.
+ if (taskgraph_rec) {
+ if (if_val == 0) {
+ taskdata->td_flags.task_serial = 1;
+ taskdata->td_flags.tiedness = TASK_TIED;
+ }
+ __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, nogroup, ub_glob,
+ num_tasks, grainsize, extras, last_chunk, tc,
+#if OMPT_SUPPORT
+ OMPT_GET_RETURN_ADDRESS(0),
+#endif
+ task_dup, taskgraph_rec);
+ // check if clause value next
+ // Also require GOMP_taskloop to reduce to linear
+ // (taskdata->td_flags.native)
+ } else
+#endif
+ if (if_val == 0) { // if(0) specified, mark task as serial
taskdata->td_flags.task_serial = 1;
taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied
// always start serial tasks linearly
- __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
- grainsize, extras, last_chunk, tc,
+ __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, nogroup, ub_glob,
+ num_tasks, grainsize, extras, last_chunk, tc,
#if OMPT_SUPPORT
OMPT_GET_RETURN_ADDRESS(0),
#endif
@@ -4939,8 +5873,8 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
"(%lld), grain %llu, extras %llu, last_chunk %lld\n",
gtid, tc, num_tasks, num_tasks_min, grainsize, extras,
last_chunk));
- __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
- grainsize, extras, last_chunk, tc,
+ __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, nogroup, ub_glob,
+ num_tasks, grainsize, extras, last_chunk, tc,
#if OMPT_SUPPORT
OMPT_GET_RETURN_ADDRESS(0),
#endif
@@ -4959,7 +5893,10 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
#if OMPT_SUPPORT && OMPT_OPTIONAL
OMPT_STORE_RETURN_ADDRESS(gtid);
#endif
- __kmpc_end_taskgroup(loc, gtid);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (!taskgraph_rec)
+#endif
+ __kmpc_end_taskgroup(loc, gtid);
}
KA_TRACE(20, ("__kmp_taskloop(exit): T#%d\n", gtid));
}
@@ -5059,3 +5996,510 @@ bool __kmpc_omp_has_task_team(kmp_int32 gtid) {
return taskdata->td_task_team != NULL;
}
+
+#if OMP_TASKGRAPH_EXPERIMENTAL
+
+// Reset a taskgraph record so that it can be reused without allocating a fresh
+// one. The KEEP_RECYCLED_DEPS flag can be set to reuse the previous record's
+// recycled dep list into the new record.
+static void __kmp_taskgraph_reset(kmp_taskgraph_record_t *rec, kmp_int32 gtid,
+ kmp_intptr_t graph_id,
+ bool keep_recycled_deps = false) {
+ rec->status = KMP_TDG_RECORDING;
+ rec->gtid = gtid;
+ rec->graph_id = graph_id;
+ rec->record_map = nullptr;
+ rec->alloc_root = nullptr;
+ // A reused record was only handed back because it had no users left, and a
+ // fresh one is raw memory, so this is an initialisation either way.
+ KMP_ATOMIC_ST_RLX(&rec->replay_users, 0);
+ rec->expired = false;
+ if (!keep_recycled_deps)
+ rec->recycled_deps = nullptr;
+ rec->num_tasks = 0;
+ rec->nodes_allocated = 0;
+ rec->num_mutexes = 0;
+ rec->node_mutexes = nullptr;
+ rec->exec_descrs = nullptr;
+ rec->num_exec_descrs = 0;
+ rec->next = nullptr;
+}
+
+static kmp_taskgraph_record_t *__kmp_taskgraph_alloc(kmp_int32 gtid,
+ kmp_intptr_t graph_id) {
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgraph_record_t *new_rec =
+ (kmp_taskgraph_record_t *)__kmp_fast_allocate(
+ thread, sizeof(kmp_taskgraph_record_t));
+ __kmp_init_lock(&new_rec->map_lock);
+ __kmp_taskgraph_reset(new_rec, gtid, graph_id,
+ /*keep_recycled_deps=*/false);
+ return new_rec;
+}
+
+// Recursively release metadata hanging off a taskgraph region (owned by the
+// region). The RECYCLED pointer is used to gather dependency-edge lists from
+// any irreducible regions found within.
+static void
+__kmp_taskgraph_free_region_metadata(kmp_info_t *thread,
+ kmp_taskgraph_region_t *region,
+ kmp_taskgraph_region_dep_t **recycled) {
+ if (recycled) {
+ __kmp_region_deplist_recycle(recycled, region->predecessors);
+ __kmp_region_deplist_recycle(recycled, region->successors);
+ } else {
+ __kmp_region_deplist_free(thread, region->predecessors);
+ __kmp_region_deplist_free(thread, region->successors);
+ }
+ region->predecessors = nullptr;
+ region->successors = nullptr;
+ if (region->reduce_input) {
+ __kmp_fast_free(thread, region->reduce_input->reduce_data);
+ __kmp_fast_free(thread, region->reduce_input);
+ }
+ if (region->mutexset) {
+ __kmp_fast_free(thread, region->mutexset);
+ }
+ switch (region->type) {
+ case TASKGRAPH_REGION_ENTRY:
+ case TASKGRAPH_REGION_EXIT:
+ case TASKGRAPH_REGION_WAIT:
+ case TASKGRAPH_REGION_NODE:
+ break;
+ case TASKGRAPH_REGION_PARALLEL:
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ case TASKGRAPH_REGION_IRREDUCIBLE: {
+ for (int k = 0; k < region->inner.num_children; k++) {
+ __kmp_taskgraph_free_region_metadata(thread, region->inner.children[k],
+ recycled);
+ }
+ break;
+ }
+ default:
+ assert(false && "unreachable");
+ }
+}
+
+static void __kmp_taskgraph_free(kmp_int32 gtid, kmp_taskgraph_record_t *rec,
+ bool keep_rec = false) {
+ kmp_info_t *thread = __kmp_threads[gtid];
+
+ if (__kmp_taskgraph_check_lifetime()) {
+ int users = KMP_ATOMIC_LD_ACQ(&rec->replay_users);
+ if (users != 0)
+ fprintf(stderr,
+ "taskgraph lifetime check: tearing down record %p with %d "
+ "replay user(s) still attached\n",
+ rec, users);
+ }
+
+ if (rec->root)
+ __kmp_taskgraph_free_region_metadata(
+ thread, rec->root, keep_rec ? &rec->recycled_deps : nullptr);
+
+ for (size_t task = 0; task < rec->num_tasks; task++) {
+ // Skip entries that don't have an associated task (e.g. taskwait nodes
+ // recorded by __kmpc_taskgraph_taskwait).
+ if (rec->record_map[task].task == nullptr)
+ continue;
+ kmp_taskdata *taskdata = KMP_TASK_TO_TASKDATA(rec->record_map[task].task);
+ // Setting this here keeps an assertion in __kmp_free_task happy: the
+ // clone may never have been replayed, in which case 'complete' will be
+ // zero here, as initialized.
+ taskdata->td_flags.complete = 1;
+ // We never decrement td_allocated_child_tasks for taskgraph tasks. This
+ // keeps another assertion happy in __kmp_free_task.
+ KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 0);
+ __kmp_free_task(gtid, taskdata, thread);
+ }
+ if (rec->record_map)
+ __kmp_thread_free(thread, rec->record_map);
+
+ // This loop frees region storage only: any linked data has already been freed
+ // by __kmp_taskgraph_free_region_metadata above.
+ kmp_taskgraph_region_t *region = rec->alloc_root;
+ while (region) {
+ kmp_taskgraph_region_t *next_region = region->alloc_chain;
+ __kmp_fast_free(thread, region);
+ region = next_region;
+ }
+
+ if (rec->exec_descrs)
+ __kmp_thread_free(thread, rec->exec_descrs);
+
+ if (rec->node_mutexes) {
+ for (int i = 0; i < rec->num_mutexes; i++)
+ __kmp_destroy_lock(&rec->node_mutexes[i]);
+ __kmp_thread_free(thread, rec->node_mutexes);
+ }
+
+ if (!keep_rec) {
+ __kmp_destroy_lock(&rec->map_lock);
+ __kmp_fast_free(thread, rec);
+ }
+}
+
+static kmp_taskgraph_header_t *__kmp_taskgraph_header_alloc(kmp_int32 gtid) {
+ kmp_taskgraph_header_t *new_hdr =
+ (kmp_taskgraph_header_t *)__kmp_allocate(sizeof(kmp_taskgraph_header_t));
+ new_hdr->first = nullptr;
+ new_hdr->expiring = nullptr;
+ __kmp_init_lock(&new_hdr->header_lock);
+ return new_hdr;
+}
+
+// Clone a (new) task that has had its private variables and shared variables
+// initialised already.
+static kmp_task_t *__kmp_taskgraph_clone_task(kmp_info_t *thread,
+ kmp_taskgraph_record_t *taskgraph,
+ kmp_task_t *orig,
+ size_t sizeof_kmp_task_t,
+ size_t sizeof_shareds) {
+ // FIXME: This should use a "taskdup" function like taskloops in cases where
+ // private variables are not trivially copyable. For now, do it by plain
+ // bitwise copy.
+ // FIXME 2: It's intended that this copy be persistent, and can be
+ // re-executed on taskgraph replay. Make sure that works (for shared
+ // variables) if stack addresses change (i.e. a task-generating function is
+ // called from different call stack depths).
+ kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(orig);
+ size_t shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
+ shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(kmp_uint64));
+ kmp_taskdata_t *copy_td = (kmp_taskdata_t *)__kmp_fast_allocate(
+ thread, shareds_offset + sizeof_shareds);
+ KMP_MEMCPY(copy_td, taskdata, shareds_offset + sizeof_shareds);
+ // Tasks cloned for a taskgraph always have this field set.
+ copy_td->owning_taskgraph = taskgraph;
+ kmp_task_t *copy_task = KMP_TASKDATA_TO_TASK(copy_td);
+ if (orig->shareds) {
+ // New task's shared data has now moved. Update the pointer.
+ copy_task->shareds = (void *)((char *)copy_td + shareds_offset);
+ }
+ KMP_ATOMIC_ST_RLX(©_td->td_incomplete_child_tasks, 0);
+ return KMP_TASKDATA_TO_TASK(copy_td);
+}
+
+// Go through list of taskgraph records and free any that are no longer being
+// executed. For now, this just frees everything in the list immediately: when
+// multiple concurrent playbacks are implemented, it should only free records
+// with zero usage counts.
+//
+// When available (and if WANT_REUSE is true), a kmp_taskgraph_record_t
+// structure is returned for reuse by a subsequent taskgraph recording.
+
+static kmp_taskgraph_record_t *__kmp_expire_taskgraph_records(
+ kmp_int32 gtid, kmp_taskgraph_record_t **expiring_p, bool want_reuse) {
+ kmp_taskgraph_record_t *record = nullptr;
+
+ while (*expiring_p) {
+ kmp_taskgraph_record_t *expiring = *expiring_p;
+ if (KMP_ATOMIC_LD_ACQ(&expiring->replay_users) != 0) {
+ if (__kmp_taskgraph_check_lifetime())
+ fprintf(stderr,
+ "taskgraph lifetime check: deferring record %p with %d "
+ "replay user(s)\n",
+ expiring, KMP_ATOMIC_LD_ACQ(&expiring->replay_users));
+ expiring_p = &expiring->next;
+ continue;
+ }
+ *expiring_p = expiring->next;
+ if (want_reuse && !record) {
+ record = expiring;
+ __kmp_taskgraph_free(gtid, record, /*keep_rec=*/true);
+ } else {
+ __kmp_taskgraph_free(gtid, expiring, /*keep_rec=*/false);
+ }
+ }
+
+ return record;
+}
+
+// Format LOC_REF as "file:line:col" for a user-facing message. The caller
+// frees the result with __kmp_str_free.
+static char *__kmp_taskgraph_src_loc(ident_t *loc_ref) {
+ if (!loc_ref || !loc_ref->psource)
+ return __kmp_str_format("unknown");
+ kmp_str_loc_t str_loc = __kmp_str_loc_init(loc_ref->psource, false);
+ char *src_loc =
+ __kmp_str_format("%s:%d:%d", str_loc.file, str_loc.line, str_loc.col);
+ __kmp_str_loc_free(&str_loc);
+ return src_loc;
+}
+
+// __kmpc_taskgraph: record or replay taskgraph
+// loc_ref: Location of TDG, not used yet
+// gtid: Global Thread ID of the encountering thread
+// tdg_handle: Handle of taskgraph -- the address of a slot in the host
+// program that we write the taskgraph (list) pointer back to.
+// graph_id: Graph ID for the taskgraph.
+// graph_reset: 1 to reset taskgraph for this taskgraph/graph_id, 0 to replay
+// (or record, initially).
+// nogroup: 1 to omit implicit taskgroup, 0 to include it.
+// entry: Pointer to the entry function
+// args: Pointer to the function arguments
+void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
+ std::atomic<void *> *tdg_handle, uintptr_t graph_id,
+ kmp_int32 graph_reset, kmp_int32 nogroup,
+ void (*entry)(void *), void *args) {
+ kmp_taskgraph_header_t *header =
+ (kmp_taskgraph_header_t *)KMP_ATOMIC_LD_ACQ(tdg_handle);
+ kmp_taskgraph_record_t *record = nullptr, **record_p = nullptr;
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgroup_t *taskgroup;
+
+ __kmpc_taskgroup(loc_ref, gtid);
+
+ taskgroup = thread->th.th_current_task->td_taskgroup;
+
+ if (!header) {
+ header = __kmp_taskgraph_header_alloc(gtid);
+ // Another thread may have allocated the header at the same time. Grab
+ // their copy if so and forget ours.
+ kmp_taskgraph_header_t *other =
+ (kmp_taskgraph_header_t *)KMP_COMPARE_AND_STORE_RET64(tdg_handle,
+ nullptr, header);
+ if (other != nullptr) {
+ __kmp_free(header);
+ header = other;
+ }
+ }
+
+ __kmp_acquire_lock(&header->header_lock, gtid);
+ for (record_p = &header->first; *record_p; record_p = &((*record_p)->next))
+ if ((*record_p)->graph_id == graph_id)
+ break;
+ record = *record_p;
+ kmp_taskgraph_record_t *reuse_record = nullptr;
+ if (record && graph_reset) {
+ kmp_taskgraph_status_t old_status = KMP_ATOMIC_LD_ACQ(&record->status);
+ // Sanity check: if the graph is not ready, it means another thread is
+ // already performing a record operation for this taskgraph/graph_id.
+ // That's likely a bug, so fail with an error.
+ if (old_status != KMP_TDG_READY) {
+ char *src_loc = __kmp_taskgraph_src_loc(loc_ref);
+ KMP_FATAL(OmpTaskgraphConcurrentRecord, src_loc, graph_id);
+ __kmp_str_free(&src_loc);
+ }
+ // Move the existing record to the header's expiring list. It can't be used
+ // for new replays from here on.
+ record->expired = true;
+ *record_p = record->next;
+ record->next = header->expiring;
+ header->expiring = record;
+ reuse_record = __kmp_expire_taskgraph_records(gtid, &header->expiring,
+ /*want_reuse=*/true);
+ record = nullptr;
+ }
+ if (!record) {
+ if (reuse_record) {
+ record = reuse_record;
+ __kmp_taskgraph_reset(record, gtid, graph_id,
+ /*keep_recycled_deps=*/true);
+ } else
+ record = __kmp_taskgraph_alloc(gtid, graph_id);
+ // We record 'nogroup' here. We always create a group for recording the
+ // taskgraph, but we could avoid doing so for replay. That's not done
+ // yet though.
+ record->nogroup_taskgroup = nogroup;
+ record->next = header->first;
+ header->first = record;
+ // Store our taskgraph record into the taskgraph directive's implicit
+ // taskgroup.
+ taskgroup->taskgraph.recording_task = thread->th.th_current_task;
+ KMP_ATOMIC_ST_REL(&taskgroup->taskgraph.recording, record);
+ }
+ // Decide between recording and replaying while the header lock is still
+ // held, and increment the usage count if we're going to replay.
+ kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&record->status);
+ if (status == KMP_TDG_READY)
+ KMP_ATOMIC_INC(&record->replay_users);
+ __kmp_release_lock(&header->header_lock, gtid);
+
+ if (status == KMP_TDG_RECORDING)
+ entry(args);
+ else if (status == KMP_TDG_READY) {
+ kmp_taskdata *current_taskdata = thread->th.th_current_task;
+ if (!record) {
+ if (__kmp_taskgraph_trace())
+ fprintf(stderr, "Skipping replay of empty taskgraph from task %p\n",
+ KMP_TASKDATA_TO_TASK(current_taskdata));
+ return;
+ }
+ if (__kmp_taskgraph_trace())
+ fprintf(stderr, "Replay taskgraph %p from task %p\n", record,
+ KMP_TASKDATA_TO_TASK(current_taskdata));
+ __kmp_acquire_lock(&record->map_lock, gtid);
+ __kmp_replay_taskgraph(gtid, current_taskdata, record, graph_id, taskgroup);
+ __kmpc_end_taskgroup(loc_ref, gtid);
+ __kmp_release_lock(&record->map_lock, gtid);
+ // Decrement usage count and expire our record if it was 'graph_reset' by
+ // some thread during this replay.
+ if (KMP_ATOMIC_DEC(&record->replay_users) == 1) {
+ __kmp_acquire_lock(&header->header_lock, gtid);
+ __kmp_expire_taskgraph_records(gtid, &header->expiring,
+ /*want_reuse=*/false);
+ __kmp_release_lock(&header->header_lock, gtid);
+ }
+ return;
+ }
+
+ __kmpc_end_taskgroup(loc_ref, gtid);
+
+ // This could perhaps be spawned as a separate task in order to avoid
+ // blocking this thread.
+ if (record->gtid == gtid) {
+ kmp_taskdata *current_taskdata = thread->th.th_current_task;
+ __kmp_build_taskgraph(gtid, current_taskdata, record);
+ }
+}
+
+kmp_uint32 __kmpc_taskgraph_task(ident_t *loc_ref, kmp_int32 gtid,
+ kmp_task_t *new_task, kmp_int32 flags,
+ size_t sizeof_kmp_task_t,
+ size_t sizeof_shareds, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list,
+ kmp_task_relocate_t relocate, void *task_dup) {
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgroup_t *taskgroup = thread->th.th_current_task->td_taskgroup;
+ kmp_taskgraph_record_t *rec =
+ __kmp_taskgraph_task_recording(thread->th.th_current_task);
+
+ if (rec) {
+ kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&rec->status);
+ if (status == KMP_TDG_RECORDING) {
+ // OpenMP 6.0 [14.3]: a detachable task must not be a replayable task in
+ // a taskgraph region. Its completion event is created when the task is
+ // generated, and a replay does not generate the task again, so there
+ // would be no event for the program to fulfil and the node would never
+ // complete. Clang rejects the lexically-nested case outright; this
+ // catches a replayable(cond) construct that reached a recording
+ // dynamically, which it cannot see.
+ if (KMP_TASK_TO_TASKDATA(new_task)->td_flags.detachable ==
+ TASK_DETACHABLE) {
+ char *src_loc = __kmp_taskgraph_src_loc(loc_ref);
+ KMP_FATAL(OmpTaskgraphDetachableTask, src_loc);
+ __kmp_str_free(&src_loc);
+ }
+ kmp_task_t *cloned_task = __kmp_taskgraph_clone_task(
+ thread, rec, new_task, sizeof_kmp_task_t, sizeof_shareds);
+ kmp_taskgraph_node_t *node = __kmp_taskgraph_node_alloc(rec, cloned_task);
+ if (taskgroup->taskgraph.reduce_input) {
+ node->reduce_input = taskgroup->taskgraph.reduce_input;
+ taskgroup->taskgraph.reduce_input = nullptr;
+ }
+#if defined(DEBUG_TASKGRAPH)
+ fprintf(stderr, "__kmpc_taskgraph_task: record task here!\n");
+ fprintf(stderr, "private size: %d, shared size: %d\n",
+ (int)(sizeof_kmp_task_t - sizeof(kmp_task_t)),
+ (int)sizeof_shareds);
+ fprintf(stderr, "ndeps: %d\n", (int)ndeps);
+ fprintf(stderr, "gtid: %d rec->gtid: %d\n", gtid, rec->gtid);
+ fprintf(stderr, "taskgroup: %p\n",
+ thread->th.th_current_task->td_taskgroup);
+ kmp_taskdata_t *parent = thread->th.th_current_task->td_parent;
+ while (parent) {
+ fprintf(stderr, " parent: %p (taskgroup %p)\n", parent,
+ parent->td_taskgroup);
+ parent = parent->td_parent;
+ }
+#endif
+ node->ndeps = ndeps;
+ node->dep_list = (kmp_depend_info_t *)__kmp_thread_malloc(
+ thread, ndeps * sizeof(kmp_depend_info_t));
+ KMP_MEMCPY(node->dep_list, dep_list, ndeps * sizeof(kmp_depend_info_t));
+ } else if (status == KMP_TDG_READY) {
+#ifdef DEBUG_TASKGRAPH
+ fprintf(stderr,
+ "non-taskgraph task entry point for task in finalized taskgraph");
+#endif
+ return 0;
+ }
+ } else {
+ kmp_taskdata_t *parent = thread->th.th_current_task->td_parent;
+ while (parent) {
+ parent = parent->td_parent;
+ }
+ }
+
+ kmp_int32 res;
+ if (ndeps == 0)
+ res = __kmpc_omp_task(loc_ref, gtid, new_task);
+ else
+ res = __kmpc_omp_task_with_deps(loc_ref, gtid, new_task, ndeps, dep_list, 0,
+ nullptr);
+
+ return res;
+}
+
+void __kmpc_taskgraph_taskwait(ident_t *loc_ref, kmp_int32 gtid,
+ kmp_int32 ndeps, kmp_depend_info_t *dep_list,
+ kmp_int32 has_no_wait) {
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgraph_record_t *rec =
+ __kmp_taskgraph_task_recording(thread->th.th_current_task);
+
+ if (rec) {
+ kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&rec->status);
+ if (status == KMP_TDG_RECORDING) {
+ kmp_taskgraph_node_t *node = __kmp_taskgraph_node_alloc(rec, nullptr);
+#ifdef DEBUG_TASKGRAPH
+ fprintf(stderr, "__kmpc_taskgraph_taskwait: record taskwait here!\n");
+ fprintf(stderr, "ndeps: %d\n", (int)ndeps);
+#endif
+ // Without `nowait` this generates an undeferred (included) task, so it
+ // is a barrier for everything recorded after it rather than a pure
+ // dependence point.
+ node->undeferred = !has_no_wait;
+ node->ndeps = ndeps;
+ node->dep_list = (kmp_depend_info_t *)__kmp_thread_malloc(
+ thread, ndeps * sizeof(kmp_depend_info_t));
+ KMP_MEMCPY(node->dep_list, dep_list, ndeps * sizeof(kmp_depend_info_t));
+ } else if (status == KMP_TDG_READY) {
+#ifdef DEBUG_TASKGRAPH
+ fprintf(stderr, "non-taskgraph taskwait entry point for taskwait in "
+ "finalized taskgraph\n");
+#endif
+ return;
+ }
+ }
+
+ __kmpc_omp_taskwait_deps_51(loc_ref, gtid, ndeps, dep_list, 0, nullptr,
+ has_no_wait);
+}
+
+kmp_uint32 __kmpc_taskgraph_taskloop(
+ ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
+ kmp_int32 if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
+ kmp_int32 nogroup, kmp_int32 sched, kmp_uint64 grainsize,
+ kmp_int32 modifier, void *task_dup, kmp_task_relocate_t relocate) {
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgraph_record_t *rec =
+ __kmp_taskgraph_task_recording(thread->th.th_current_task);
+
+ if (rec) {
+ kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&rec->status);
+ if (status == KMP_TDG_RECORDING)
+ __kmp_taskloop(loc_ref, gtid, new_task, if_val, lb, ub, st, nogroup,
+ sched, grainsize, modifier, task_dup, rec);
+ else if (status == KMP_TDG_READY) {
+#ifdef DEBUG_TASKGRAPH
+ fprintf(stderr, "non-taskgraph taskloop entry point for taskloop in "
+ "finalized taskgraph\n");
+#endif
+ return 0;
+ }
+ }
+
+ // For 'nogroup' here, we pass TRUE because this entry point does want to
+ // know if we originally had the 'nogroup' clause or not -- but the group is
+ // created using separate API calls wrapping this one (or __kmpc_taskloop).
+ // We don't want to create another taskgroup in __kmp_taskloop here in any
+ // case.
+ __kmp_taskloop(loc_ref, gtid, new_task, if_val, lb, ub, st, /*nogroup=*/true,
+ sched, grainsize, modifier, task_dup);
+
+ return 0;
+}
+
+#endif
>From 396023a1f03cff77b8151eb45ab54b3daa29e3a9 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 27 Mar 2026 12:54:04 -0500
Subject: [PATCH 03/29] [Clang][OpenMP] Add OpenMP 6.0 taskgraph
parsing/trivial semantics
This patch adds OpenMP 6.0 'taskgraph' support, as a no-op: i.e. the
contents of the taskgraph region are just executed as if the directive
was omitted.
Co-Authored-By: Adrian Munera <adrian.munera at bsc.es>
Co-Authored-By: Jose M Monsalve Diaz <JoseM.MonsalveDiaz at amd.com>
Pull Request: https://github.com/llvm/llvm-project/pull/194050
---
clang/bindings/python/clang/cindex.py | 3 ++
clang/include/clang-c/Index.h | 4 ++
clang/include/clang/AST/RecursiveASTVisitor.h | 3 ++
clang/include/clang/AST/StmtOpenMP.h | 49 +++++++++++++++++++
clang/include/clang/Basic/StmtNodes.td | 1 +
clang/include/clang/Sema/SemaOpenMP.h | 4 ++
.../include/clang/Serialization/ASTBitCodes.h | 1 +
clang/lib/AST/StmtOpenMP.cpp | 15 ++++++
clang/lib/AST/StmtPrinter.cpp | 5 ++
clang/lib/AST/StmtProfile.cpp | 4 ++
clang/lib/Basic/OpenMPKinds.cpp | 4 +-
clang/lib/CodeGen/CGOpenMPRuntime.cpp | 40 +++++++++++++++
clang/lib/CodeGen/CGOpenMPRuntime.h | 13 +++++
clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp | 2 +
clang/lib/CodeGen/CGStmt.cpp | 3 ++
clang/lib/CodeGen/CGStmtOpenMP.cpp | 15 ++++++
clang/lib/CodeGen/CodeGenFunction.h | 1 +
clang/lib/Sema/SemaExceptionSpec.cpp | 1 +
clang/lib/Sema/SemaOpenMP.cpp | 41 ++++++++++++++--
clang/lib/Sema/TreeTransform.h | 11 +++++
clang/lib/Serialization/ASTReaderStmt.cpp | 10 ++++
clang/lib/Serialization/ASTWriterStmt.cpp | 6 +++
clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 1 +
clang/test/OpenMP/taskgraph_bad_clauses.cpp | 19 +++++++
.../OpenMP/unimplemented_clause_messages.cpp | 1 -
clang/tools/libclang/CIndex.cpp | 2 +
clang/tools/libclang/CXCursor.cpp | 3 ++
27 files changed, 256 insertions(+), 6 deletions(-)
create mode 100644 clang/test/OpenMP/taskgraph_bad_clauses.cpp
diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 0a3e7de7a884ae..b0ef3f008c7839 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1475,6 +1475,9 @@ def is_unexposed(self):
# OpenMP flatten directive.
OMP_FLATTEN_DIRECTIVE = 314
+ # OpenMP taskgraph directive.
+ OMP_TASKGRAPH_DIRECTIVE = 315
+
# OpenACC Compute Construct.
OPEN_ACC_COMPUTE_DIRECTIVE = 320
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index f83db6a428ccf2..ee7372aea35285 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2178,6 +2178,10 @@ enum CXCursorKind {
*/
CXCursor_OMPFlattenDirective = 314,
+ /** OpenMP taskgraph directive.
+ */
+ CXCursor_OMPTaskgraphDirective = 315,
+
/** OpenACC Compute Construct.
*/
CXCursor_OpenACCComputeConstruct = 320,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 13dd5eda84fa65..809435580708dc 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3306,6 +3306,9 @@ DEF_TRAVERSE_STMT(OMPBarrierDirective,
DEF_TRAVERSE_STMT(OMPTaskwaitDirective,
{ TRY_TO(TraverseOMPExecutableDirective(S)); })
+DEF_TRAVERSE_STMT(OMPTaskgraphDirective,
+ { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
DEF_TRAVERSE_STMT(OMPTaskgroupDirective,
{ TRY_TO(TraverseOMPExecutableDirective(S)); })
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index a070d97a6db1f3..70728d25ce7def 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -2811,6 +2811,55 @@ class OMPTaskwaitDirective : public OMPExecutableDirective {
}
};
+/// This represents '#pragma omp taskgraph' directive.
+/// Available with OpenMP 6.0.
+///
+/// \code
+/// #pragma omp taskgraph
+/// \endcode
+///
+class OMPTaskgraphDirective final : public OMPExecutableDirective {
+ friend class ASTStmtReader;
+ friend class OMPExecutableDirective;
+ /// Build directive with the given start and end location.
+ ///
+ /// \param StartLoc Starting location of the directive kind.
+ /// \param EndLoc Ending location of the directive.
+ ///
+ OMPTaskgraphDirective(SourceLocation StartLoc, SourceLocation EndLoc)
+ : OMPExecutableDirective(OMPTaskgraphDirectiveClass,
+ llvm::omp::OMPD_taskgraph, StartLoc, EndLoc) {}
+
+ /// Build an empty directive.
+ ///
+ explicit OMPTaskgraphDirective()
+ : OMPExecutableDirective(OMPTaskgraphDirectiveClass,
+ llvm::omp::OMPD_taskgraph, SourceLocation(),
+ SourceLocation()) {}
+
+public:
+ /// Creates directive.
+ ///
+ /// \param C AST context.
+ /// \param StartLoc Starting location of the directive kind.
+ /// \param EndLoc Ending Location of the directive.
+ ///
+ static OMPTaskgraphDirective *
+ Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+ ArrayRef<OMPClause *> Clauses, Stmt *AssociatedStmt);
+
+ /// Creates an empty directive.
+ ///
+ /// \param C AST context.
+ ///
+ static OMPTaskgraphDirective *CreateEmpty(const ASTContext &C,
+ unsigned NumClauses, EmptyShell);
+
+ static bool classof(const Stmt *T) {
+ return T->getStmtClass() == OMPTaskgraphDirectiveClass;
+ }
+};
+
/// This represents '#pragma omp taskgroup' directive.
///
/// \code
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index 96f7a0080f20d6..bc52e8a6597221 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -276,6 +276,7 @@ def OMPTaskDirective : StmtNode<OMPExecutableDirective>;
def OMPTaskyieldDirective : StmtNode<OMPExecutableDirective>;
def OMPBarrierDirective : StmtNode<OMPExecutableDirective>;
def OMPTaskwaitDirective : StmtNode<OMPExecutableDirective>;
+def OMPTaskgraphDirective : StmtNode<OMPExecutableDirective>;
def OMPTaskgroupDirective : StmtNode<OMPExecutableDirective>;
def OMPFlushDirective : StmtNode<OMPExecutableDirective>;
def OMPDepobjDirective : StmtNode<OMPExecutableDirective>;
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 49d504e31bdb87..c4d7bef5dd04cb 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -584,6 +584,10 @@ class SemaOpenMP : public SemaBase {
/// Called on well-formed '\#pragma omp barrier'.
StmtResult ActOnOpenMPBarrierDirective(SourceLocation StartLoc,
SourceLocation EndLoc);
+ /// Called on well-formed '\#pragma omp taskgraph'.
+ StmtResult ActOnOpenMPTaskgraphDirective(ArrayRef<OMPClause *> Clauses,
+ Stmt *AStmt, SourceLocation StartLoc,
+ SourceLocation EndLoc);
/// Called on well-formed '\#pragma omp taskwait'.
StmtResult ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
SourceLocation StartLoc,
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index ae3a2fad6ebe7f..bf3dbc692c0b2f 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -2016,6 +2016,7 @@ enum StmtCode {
STMT_OMP_ERROR_DIRECTIVE,
STMT_OMP_BARRIER_DIRECTIVE,
STMT_OMP_TASKWAIT_DIRECTIVE,
+ STMT_OMP_TASKGRAPH_DIRECTIVE,
STMT_OMP_FLUSH_DIRECTIVE,
STMT_OMP_DEPOBJ_DIRECTIVE,
STMT_OMP_SCAN_DIRECTIVE,
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 5ba11b2d8c42fa..ef6755b66d1c28 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -1057,6 +1057,21 @@ OMPTaskwaitDirective *OMPTaskwaitDirective::CreateEmpty(const ASTContext &C,
return createEmptyDirective<OMPTaskwaitDirective>(C, NumClauses);
}
+OMPTaskgraphDirective *OMPTaskgraphDirective::Create(
+ const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+ ArrayRef<OMPClause *> Clauses, Stmt *AssociatedStmt) {
+ auto *Dir = createDirective<OMPTaskgraphDirective>(
+ C, Clauses, AssociatedStmt, /*NumChildren=*/0, StartLoc, EndLoc);
+ return Dir;
+}
+
+OMPTaskgraphDirective *OMPTaskgraphDirective::CreateEmpty(const ASTContext &C,
+ unsigned NumClauses,
+ EmptyShell) {
+ return createEmptyDirective<OMPTaskgraphDirective>(
+ C, NumClauses, /*HasAssociatedStmt=*/true);
+}
+
OMPTaskgroupDirective *OMPTaskgroupDirective::Create(
const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
ArrayRef<OMPClause *> Clauses, Stmt *AssociatedStmt, Expr *ReductionRef) {
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 4f184868659f48..5527fd9f1204fa 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -948,6 +948,11 @@ void StmtPrinter::VisitOMPAssumeDirective(OMPAssumeDirective *Node) {
PrintOMPExecutableDirective(Node);
}
+void StmtPrinter::VisitOMPTaskgraphDirective(OMPTaskgraphDirective *Node) {
+ Indent() << "#pragma omp taskgraph";
+ PrintOMPExecutableDirective(Node);
+}
+
void StmtPrinter::VisitOMPErrorDirective(OMPErrorDirective *Node) {
Indent() << "#pragma omp error";
PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index b24aaf2b9cc422..f276647c71cb72 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1194,6 +1194,10 @@ void StmtProfiler::VisitOMPAssumeDirective(const OMPAssumeDirective *S) {
VisitOMPExecutableDirective(S);
}
+void StmtProfiler::VisitOMPTaskgraphDirective(const OMPTaskgraphDirective *S) {
+ VisitOMPExecutableDirective(S);
+}
+
void StmtProfiler::VisitOMPErrorDirective(const OMPErrorDirective *S) {
VisitOMPExecutableDirective(S);
}
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 8049e5d7c1be9c..ca0d3a0f6b84e4 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -996,6 +996,9 @@ void clang::getOpenMPCaptureRegions(
case OMPD_teams:
CaptureRegions.push_back(OMPD_teams);
break;
+ case OMPD_taskgraph:
+ CaptureRegions.push_back(OMPD_taskgraph);
+ break;
case OMPD_taskloop:
CaptureRegions.push_back(OMPD_taskloop);
break;
@@ -1019,7 +1022,6 @@ void clang::getOpenMPCaptureRegions(
case OMPD_simd:
case OMPD_single:
case OMPD_target_data:
- case OMPD_taskgraph:
case OMPD_taskgroup:
case OMPD_stripe:
// These directives (when standalone) use OMPD_unknown as the region,
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index d3a6ffee3ea748..45eac2df8af3f1 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -64,6 +64,8 @@ class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
ParallelOutlinedRegion,
/// Region with outlined function for standalone 'task' directive.
TaskOutlinedRegion,
+ /// Region with outlined function for standalone 'taskgraph' directive.
+ TaskgraphOutlinedRegion,
/// Region for constructs that do not require function outlining,
/// like 'for', 'sections', 'atomic' etc. directives.
InlinedRegion,
@@ -2265,6 +2267,33 @@ void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
Region->emitUntiedSwitch(CGF);
}
+void CGOpenMPRuntime::emitTaskgraphCall(CodeGenFunction &CGF,
+ SourceLocation Loc,
+ const OMPExecutableDirective &D,
+ const Expr *IfCond) {
+ if (!CGF.HaveInsertPoint())
+ return;
+
+ CodeGenFunction OutlinedCGF(CGM, /*suppressNewContext=*/true);
+
+ const auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
+ LValue CapStruct = CGF.InitCapturedStruct(*CS);
+
+ llvm::Function *OutlinedFn = OutlinedCGF.GenerateCapturedStmtFunction(*CS);
+
+ llvm::Value *CapturedArgsPtr =
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CapStruct.getPointer(OutlinedCGF), CGM.VoidPtrTy);
+
+ auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
+ Action.Enter(CGF);
+ CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn,
+ CapturedArgsPtr);
+ };
+ RegionCodeGenTy RCG(CodeGen);
+ RCG(CGF);
+}
+
void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
const RegionCodeGenTy &TaskgroupOpGen,
SourceLocation Loc) {
@@ -6776,6 +6805,7 @@ const Expr *CGOpenMPRuntime::getNumTeamsExprForTargetDirective(
case OMPD_taskyield:
case OMPD_barrier:
case OMPD_taskwait:
+ case OMPD_taskgraph:
case OMPD_taskgroup:
case OMPD_atomic:
case OMPD_flush:
@@ -10763,6 +10793,7 @@ getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
case OMPD_taskyield:
case OMPD_barrier:
case OMPD_taskwait:
+ case OMPD_taskgraph:
case OMPD_taskgroup:
case OMPD_atomic:
case OMPD_flush:
@@ -11529,6 +11560,7 @@ void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
case OMPD_taskyield:
case OMPD_barrier:
case OMPD_taskwait:
+ case OMPD_taskgraph:
case OMPD_taskgroup:
case OMPD_atomic:
case OMPD_flush:
@@ -12102,6 +12134,7 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
case OMPD_taskyield:
case OMPD_barrier:
case OMPD_taskwait:
+ case OMPD_taskgraph:
case OMPD_taskgroup:
case OMPD_atomic:
case OMPD_flush:
@@ -13456,6 +13489,13 @@ void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
llvm_unreachable("Not supported in SIMD-only mode");
}
+void CGOpenMPSIMDRuntime::emitTaskgraphCall(CodeGenFunction &CGF,
+ SourceLocation Loc,
+ const OMPExecutableDirective &D,
+ const Expr *IfCond) {
+ llvm_unreachable("Not supported in SIMD-only mode");
+}
+
void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
SourceLocation Loc) {
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index ae55295cadc14f..8d7094afa702f1 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -1380,6 +1380,11 @@ class CGOpenMPRuntime {
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
const OMPTaskDataTy &Data);
+ /// Emit code for 'taskgraph' directive.
+ virtual void emitTaskgraphCall(CodeGenFunction &CGF, SourceLocation Loc,
+ const OMPExecutableDirective &D,
+ const Expr *IfCond);
+
/// Emit code for 'cancellation point' construct.
/// \param CancelRegion Region kind for which the cancellation point must be
/// emitted.
@@ -2215,6 +2220,14 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
const OMPTaskDataTy &Data) override;
+ /// Emit code for 'taskgraph' directive.
+ /// \param IfCond Expression evaluated in if clause associated with the
+ // taskgraph.
+ /// \param D Directive to emit.
+ void emitTaskgraphCall(CodeGenFunction &CGF, SourceLocation Loc,
+ const OMPExecutableDirective &D,
+ const Expr *IfCond) override;
+
/// Emit code for 'cancellation point' construct.
/// \param CancelRegion Region kind for which the cancellation point must be
/// emitted.
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index e7322bdd0deca9..b589f73136de40 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -576,6 +576,7 @@ static bool hasNestedSPMDDirective(ASTContext &Ctx,
case OMPD_taskyield:
case OMPD_barrier:
case OMPD_taskwait:
+ case OMPD_taskgraph:
case OMPD_taskgroup:
case OMPD_atomic:
case OMPD_flush:
@@ -665,6 +666,7 @@ static bool supportsSPMDExecutionMode(ASTContext &Ctx,
case OMPD_taskyield:
case OMPD_barrier:
case OMPD_taskwait:
+ case OMPD_taskgraph:
case OMPD_taskgroup:
case OMPD_atomic:
case OMPD_flush:
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 05737ce071647d..05b674dbe7b5ab 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -296,6 +296,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
case Stmt::OMPTaskwaitDirectiveClass:
EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S));
break;
+ case Stmt::OMPTaskgraphDirectiveClass:
+ EmitOMPTaskgraphDirective(cast<OMPTaskgraphDirective>(*S));
+ break;
case Stmt::OMPTaskgroupDirectiveClass:
EmitOMPTaskgroupDirective(cast<OMPTaskgroupDirective>(*S));
break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index e63957df3e4069..91a85b4c26c338 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -1895,6 +1895,7 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
case OMPD_error:
case OMPD_barrier:
case OMPD_taskwait:
+ case OMPD_taskgraph:
case OMPD_taskgroup:
case OMPD_flush:
case OMPD_depobj:
@@ -6175,6 +6176,20 @@ void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);
}
+void CodeGenFunction::EmitOMPTaskgraphDirective(
+ const OMPTaskgraphDirective &S) {
+ const Expr *IfCond = nullptr;
+ for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
+ if (C->getNameModifier() == OMPD_unknown ||
+ C->getNameModifier() == OMPD_taskgraph) {
+ IfCond = C->getCondition();
+ break;
+ }
+ }
+
+ CGM.getOpenMPRuntime().emitTaskgraphCall(*this, S.getBeginLoc(), S, IfCond);
+}
+
static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T) {
return T.clauses().empty();
}
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index e17ed28c5d4f3a..e972d9b1cc1b32 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4048,6 +4048,7 @@ class CodeGenFunction : public CodeGenTypeCache {
void EmitOMPErrorDirective(const OMPErrorDirective &S);
void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
+ void EmitOMPTaskgraphDirective(const OMPTaskgraphDirective &S);
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
void EmitOMPFlushDirective(const OMPFlushDirective &S);
void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 56fe626f8192fe..67f6f08a625bcc 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1533,6 +1533,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
case Stmt::OMPScopeDirectiveClass:
case Stmt::OMPTaskDirectiveClass:
case Stmt::OMPTaskgroupDirectiveClass:
+ case Stmt::OMPTaskgraphDirectiveClass:
case Stmt::OMPTaskLoopDirectiveClass:
case Stmt::OMPTaskLoopSimdDirectiveClass:
case Stmt::OMPTaskwaitDirectiveClass:
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 5fba2895938703..e52cae2a056f78 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4899,6 +4899,14 @@ getUnknownRegionParams(Sema &SemaRef) {
return Params;
}
+static SmallVector<SemaOpenMP::CapturedParamNameType>
+getTaskgraphRegionParams(Sema &SemaRef) {
+ SmallVector<SemaOpenMP::CapturedParamNameType> Params{
+ std::make_pair(StringRef(), QualType()) // __context with shared vars
+ };
+ return Params;
+}
+
static SmallVector<SemaOpenMP::CapturedParamNameType>
getTaskloopRegionParams(Sema &SemaRef) {
ASTContext &Context = SemaRef.getASTContext();
@@ -4972,6 +4980,10 @@ static void processCapturedRegions(Sema &SemaRef, OpenMPDirectiveKind DKind,
// function directly.
MarkAsInlined(SemaRef.getCurCapturedRegion());
break;
+ case OMPD_taskgraph:
+ SemaRef.ActOnCapturedRegionStart(
+ Loc, CurScope, CR_OpenMP, getTaskgraphRegionParams(SemaRef), Level);
+ break;
case OMPD_target:
SemaRef.ActOnCapturedRegionStart(Loc, CurScope, CR_OpenMP,
getTargetRegionParams(SemaRef), Level);
@@ -6977,6 +6989,12 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
"No associated statement allowed for 'omp taskwait' directive");
Res = ActOnOpenMPTaskwaitDirective(ClausesWithImplicit, StartLoc, EndLoc);
break;
+ case OMPD_taskgraph:
+ assert(AStmt &&
+ "Associated statement required for 'omp taskgraph' directive");
+ Res = ActOnOpenMPTaskgraphDirective(ClausesWithImplicit, AStmt, StartLoc,
+ EndLoc);
+ break;
case OMPD_taskgroup:
Res = ActOnOpenMPTaskgroupDirective(ClausesWithImplicit, AStmt, StartLoc,
EndLoc);
@@ -7191,10 +7209,6 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
case OMPD_begin_declare_variant:
case OMPD_end_declare_variant:
llvm_unreachable("OpenMP Directive is not allowed");
- case OMPD_taskgraph:
- Diag(StartLoc, diag::err_omp_unexpected_directive)
- << 1 << getOpenMPDirectiveName(OMPD_taskgraph);
- return StmtError();
case OMPD_unknown:
default:
llvm_unreachable("Unknown OpenMP directive");
@@ -12393,6 +12407,25 @@ SemaOpenMP::ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
Clauses);
}
+StmtResult
+SemaOpenMP::ActOnOpenMPTaskgraphDirective(ArrayRef<OMPClause *> Clauses,
+ Stmt *AStmt, SourceLocation StartLoc,
+ SourceLocation EndLoc) {
+ llvm::omp::Version OMPVersion = getLangOpts().getOpenMPVersion();
+ if (!OMPVersion || OMPVersion < 60) {
+ Diag(StartLoc, diag::err_omp_unexpected_directive)
+ << 1 << getOpenMPDirectiveName(OMPD_taskgraph, OMPVersion);
+ return StmtError();
+ }
+ if (!AStmt)
+ return StmtError();
+
+ assert(isa<CapturedStmt>(AStmt) && "Captured statement expected");
+
+ return OMPTaskgraphDirective::Create(getASTContext(), StartLoc, EndLoc,
+ Clauses, AStmt);
+}
+
StmtResult
SemaOpenMP::ActOnOpenMPTaskgroupDirective(ArrayRef<OMPClause *> Clauses,
Stmt *AStmt, SourceLocation StartLoc,
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 7b5ee7b7068ae4..390ed4835e5364 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -10387,6 +10387,17 @@ TreeTransform<Derived>::TransformOMPAssumeDirective(OMPAssumeDirective *D) {
return Res;
}
+template <typename Derived>
+StmtResult TreeTransform<Derived>::TransformOMPTaskgraphDirective(
+ OMPTaskgraphDirective *D) {
+ DeclarationNameInfo DirName;
+ getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+ OMPD_taskgraph, DirName, nullptr, D->getBeginLoc());
+ StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+ getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+ return Res;
+}
+
template <typename Derived>
StmtResult
TreeTransform<Derived>::TransformOMPErrorDirective(OMPErrorDirective *D) {
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 26aee27c65e82f..57fc6add3d91d8 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2700,6 +2700,11 @@ void ASTStmtReader::VisitOMPAssumeDirective(OMPAssumeDirective *D) {
VisitOMPExecutableDirective(D);
}
+void ASTStmtReader::VisitOMPTaskgraphDirective(OMPTaskgraphDirective *D) {
+ VisitStmt(D);
+ VisitOMPExecutableDirective(D);
+}
+
void ASTStmtReader::VisitOMPErrorDirective(OMPErrorDirective *D) {
VisitStmt(D);
// The NumClauses field was read in ReadStmtFromStream.
@@ -3895,6 +3900,11 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
Context, Record[ASTStmtReader::NumStmtFields], Empty);
break;
+ case STMT_OMP_TASKGRAPH_DIRECTIVE:
+ S = OMPTaskgraphDirective::CreateEmpty(
+ Context, Record[ASTStmtReader::NumStmtFields], Empty);
+ break;
+
case STMT_OMP_ERROR_DIRECTIVE:
S = OMPErrorDirective::CreateEmpty(
Context, Record[ASTStmtReader::NumStmtFields], Empty);
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index c6f04336f806ee..f045e2ceffdd84 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2849,6 +2849,12 @@ void ASTStmtWriter::VisitOMPAssumeDirective(OMPAssumeDirective *D) {
Code = serialization::STMT_OMP_ASSUME_DIRECTIVE;
}
+void ASTStmtWriter::VisitOMPTaskgraphDirective(OMPTaskgraphDirective *D) {
+ VisitStmt(D);
+ VisitOMPExecutableDirective(D);
+ Code = serialization::STMT_OMP_TASKGRAPH_DIRECTIVE;
+}
+
void ASTStmtWriter::VisitOMPErrorDirective(OMPErrorDirective *D) {
VisitStmt(D);
Record.push_back(D->getNumClauses());
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index a7257126d9a415..b042571b0cd710 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1893,6 +1893,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
case Stmt::OMPTaskyieldDirectiveClass:
case Stmt::OMPBarrierDirectiveClass:
case Stmt::OMPTaskwaitDirectiveClass:
+ case Stmt::OMPTaskgraphDirectiveClass:
case Stmt::OMPErrorDirectiveClass:
case Stmt::OMPTaskgroupDirectiveClass:
case Stmt::OMPFlushDirectiveClass:
diff --git a/clang/test/OpenMP/taskgraph_bad_clauses.cpp b/clang/test/OpenMP/taskgraph_bad_clauses.cpp
new file mode 100644
index 00000000000000..16b38a35fb4bd4
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_bad_clauses.cpp
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -verify=expected,omp51 -fopenmp -fopenmp-version=51 -ferror-limit 100 -o - %s
+// RUN: %clang_cc1 -verify=expected,omp60 -fopenmp -fopenmp-version=60 -ferror-limit 100 -o - %s
+
+int main() {
+ int data[10];
+#pragma omp taskgraph map(tofrom: data[0:10]) // expected-error {{unexpected OpenMP clause 'map' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+ {
+ }
+#pragma omp taskgraph depend(inout: data) // expected-error {{unexpected OpenMP clause 'depend' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+ {
+ }
+#pragma omp taskgraph if(taskgraph: 1) // omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+ {
+ }
+#pragma omp taskgraph if(cancel: 1) // omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}} expected-error {{directive name modifier 'cancel' is not allowed for '#pragma omp taskgraph'}}
+ {
+ }
+ return 0;
+}
diff --git a/clang/test/OpenMP/unimplemented_clause_messages.cpp b/clang/test/OpenMP/unimplemented_clause_messages.cpp
index 0f51c916093fff..b83b9e9195a8ec 100644
--- a/clang/test/OpenMP/unimplemented_clause_messages.cpp
+++ b/clang/test/OpenMP/unimplemented_clause_messages.cpp
@@ -76,7 +76,6 @@ void processPointsInLine() {
// Make sure test doesn't crash.
void test_tasgraph()
{
- // omp60-error at +2{{unexpected OpenMP directive '#pragma omp taskgraph'}}
// omp51-error at +1{{unexpected OpenMP directive '#pragma omp taskgraph'}}
#pragma omp taskgraph
for (int i = 0; i < 10; ++i)
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 8d246d41f31f66..0e2a411f6334c8 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -6419,6 +6419,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
return cxstring::createRef("OMPTaskwaitDirective");
case CXCursor_OMPAssumeDirective:
return cxstring::createRef("OMPAssumeDirective");
+ case CXCursor_OMPTaskgraphDirective:
+ return cxstring::createRef("OMPTaskgraphDirective");
case CXCursor_OMPErrorDirective:
return cxstring::createRef("OMPErrorDirective");
case CXCursor_OMPTaskgroupDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index e042e12ea4c062..9296f352774eda 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -764,6 +764,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
case Stmt::OMPTaskwaitDirectiveClass:
K = CXCursor_OMPTaskwaitDirective;
break;
+ case Stmt::OMPTaskgraphDirectiveClass:
+ K = CXCursor_OMPTaskgraphDirective;
+ break;
case Stmt::OMPErrorDirectiveClass:
K = CXCursor_OMPErrorDirective;
break;
>From 3fae1dcde03b22c96a0773ad044b0e48cc2110e4 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 27 Mar 2026 09:40:01 -0500
Subject: [PATCH 04/29] [Clang][OpenMP] Add graph_id and graph_reset clauses
for 'taskgraph'
This patch adds support for the graph_id and graph_reset clauses for
the OpenMP 6.0 "taskgraph" directive. The expression may be omitted
for the graph_reset clause (which seems of limited use, but is allowed
by the spec), so that class does not inherit from OMPOneStmtClause.
Tests are added to cover serialization/deserialization, OpenMP version
restrictions, and so on.
Assisted-By: Codex
Assisted-By: Claude
Co-Authored-By: Josep Pinot <josep.pinot at bsc.es>
Pull Request: https://github.com/llvm/llvm-project/pull/194048
---
clang/include/clang/AST/OpenMPClause.h | 95 +++++++++++++++++++
clang/include/clang/AST/RecursiveASTVisitor.h | 13 +++
clang/include/clang/Sema/SemaOpenMP.h | 9 ++
clang/lib/AST/OpenMPClause.cpp | 26 +++++
clang/lib/AST/StmtProfile.cpp | 8 ++
clang/lib/Basic/OpenMPKinds.cpp | 4 +
clang/lib/Parse/ParseOpenMP.cpp | 15 +++
clang/lib/Sema/SemaOpenMP.cpp | 60 ++++++++++++
clang/lib/Sema/TreeTransform.h | 43 +++++++++
clang/lib/Serialization/ASTReader.cpp | 16 ++++
clang/lib/Serialization/ASTWriter.cpp | 10 ++
clang/test/OpenMP/taskgraph_ast_print.cpp | 31 ++++++
.../OpenMP/taskgraph_clauses_ast_print.cpp | 94 ++++++++++++++++++
.../OpenMP/taskgraph_clauses_messages.cpp | 36 +++++++
clang/tools/libclang/CIndex.cpp | 6 ++
llvm/include/llvm/Frontend/OpenMP/OMP.td | 2 +
16 files changed, 468 insertions(+)
create mode 100644 clang/test/OpenMP/taskgraph_ast_print.cpp
create mode 100644 clang/test/OpenMP/taskgraph_clauses_ast_print.cpp
create mode 100644 clang/test/OpenMP/taskgraph_clauses_messages.cpp
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 960dcf65b2ca99..e046e639c1511d 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -8816,6 +8816,101 @@ class OMPIsDevicePtrClause final
}
};
+/// This represents clause 'graph_id' in the '#pragma omp taskgraph"
+/// directives.
+///
+/// \code
+/// #pragma omp taskgraph graph_id(a)
+class OMPGraphIdClause final
+ : public OMPOneStmtClause<llvm::omp::OMPC_graph_id, OMPClause> {
+ friend class OMPClauseReader;
+
+ /// Set condition.
+ void setId(Expr *Id) { setStmt(Id); }
+
+public:
+ /// Build 'graph_id' clause with identifier value \a Id.
+ ///
+ /// \param Id Id value for the clause.
+ /// \param StartLoc Starting location of the clause.
+ /// \param LParenLoc Location of '('.
+ /// \param EndLoc Ending location of the clause.
+ OMPGraphIdClause(Expr *Id, SourceLocation StartLoc, SourceLocation LParenLoc,
+ SourceLocation EndLoc)
+ : OMPOneStmtClause(Id, StartLoc, LParenLoc, EndLoc) {}
+
+ /// Build an empty clause.
+ OMPGraphIdClause() : OMPOneStmtClause() {}
+
+ /// Returns condition.
+ Expr *getId() const { return getStmtAs<Expr>(); }
+};
+
+// This represents clause 'graph_reset' in the '#pragma omp taskgraph"
+/// directives.
+///
+/// \code
+/// #pragma omp taskgraph graph_reset(true)
+class OMPGraphResetClause final : public OMPClause {
+ friend class OMPClauseReader;
+
+ /// Location of '('.
+ SourceLocation LParenLoc;
+
+ /// Condition of the 'graph_reset' clause.
+ Stmt *Condition = nullptr;
+
+public:
+ /// Build 'graph_reset' clause with condition \a Cond.
+ ///
+ /// \param Cond Condition of the clause.
+ /// \param StartLoc Starting location of the clause.
+ /// \param LParenLoc Location of '('.
+ /// \param EndLoc Ending location of the clause.
+ OMPGraphResetClause(Expr *Cond, SourceLocation StartLoc,
+ SourceLocation LParenLoc, SourceLocation EndLoc)
+ : OMPClause(llvm::omp::OMPC_graph_reset, StartLoc, EndLoc),
+ LParenLoc(LParenLoc), Condition(Cond) {}
+
+ /// Build an empty clause.
+ OMPGraphResetClause()
+ : OMPClause(llvm::omp::OMPC_graph_reset, SourceLocation(),
+ SourceLocation()) {}
+
+ /// Sets the location of '('.
+ void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+ /// Returns the location of '('.
+ SourceLocation getLParenLoc() const { return LParenLoc; }
+
+ /// Set condition.
+ void setCondition(Expr *Cond) { Condition = Cond; }
+
+ /// Returns condition.
+ Expr *getCondition() const { return cast_or_null<Expr>(Condition); }
+
+ child_range children() {
+ if (Condition)
+ return child_range(&Condition, &Condition + 1);
+ return child_range(child_iterator(), child_iterator());
+ }
+
+ const_child_range children() const {
+ if (Condition)
+ return const_child_range(&Condition, &Condition + 1);
+ return const_child_range(const_child_iterator(), const_child_iterator());
+ }
+
+ child_range used_children();
+ const_child_range used_children() const {
+ return const_cast<OMPGraphResetClause *>(this)->used_children();
+ }
+
+ static bool classof(const OMPClause *T) {
+ return T->getClauseKind() == llvm::omp::OMPC_graph_reset;
+ }
+};
+
/// This represents clause 'has_device_ptr' in the '#pragma omp ...'
/// directives.
///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 809435580708dc..1913d095622f0e 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -4209,6 +4209,19 @@ bool RecursiveASTVisitor<Derived>::VisitOMPIsDevicePtrClause(
return true;
}
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPGraphIdClause(OMPGraphIdClause *C) {
+ TRY_TO(TraverseStmt(C->getId()));
+ return true;
+}
+
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPGraphResetClause(
+ OMPGraphResetClause *C) {
+ TRY_TO(TraverseStmt(C->getCondition()));
+ return true;
+}
+
template <typename Derived>
bool RecursiveASTVisitor<Derived>::VisitOMPHasDeviceAddrClause(
OMPHasDeviceAddrClause *C) {
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index c4d7bef5dd04cb..ea601759b47217 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -987,6 +987,15 @@ class SemaOpenMP : public SemaBase {
ActOnOpenMPOrderedClause(SourceLocation StartLoc, SourceLocation EndLoc,
SourceLocation LParenLoc = SourceLocation(),
Expr *NumForLoops = nullptr);
+ /// Called on well-formed 'graph_id' clause.
+ OMPClause *ActOnOpenMPGraphIdClause(Expr *Id, SourceLocation StartLoc,
+ SourceLocation LParenLoc,
+ SourceLocation EndLoc);
+ /// Called on well-formed 'graph_reset' clause.
+ OMPClause *ActOnOpenMPGraphResetClause(Expr *Condition,
+ SourceLocation StartLoc,
+ SourceLocation LParenLoc,
+ SourceLocation EndLoc);
/// Called on well-formed 'grainsize' clause.
OMPClause *ActOnOpenMPGrainsizeClause(OpenMPGrainsizeClauseModifier Modifier,
Expr *Size, SourceLocation StartLoc,
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index f0102720f7d33b..82e617f7b5cf0c 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -265,6 +265,8 @@ const OMPClauseWithPostUpdate *OMPClauseWithPostUpdate::get(const OMPClause *C)
case OMPC_thread_limit:
case OMPC_priority:
case OMPC_grainsize:
+ case OMPC_graph_id:
+ case OMPC_graph_reset:
case OMPC_nogroup:
case OMPC_num_tasks:
case OMPC_hint:
@@ -339,6 +341,12 @@ OMPClause::child_range OMPGrainsizeClause::used_children() {
return child_range(&Grainsize, &Grainsize + 1);
}
+OMPClause::child_range OMPGraphResetClause::used_children() {
+ if (Condition)
+ return child_range(&Condition, &Condition + 1);
+ return children();
+}
+
OMPClause::child_range OMPNumTasksClause::used_children() {
if (Stmt **C = getAddrOfExprAsWritten(getPreInitStmt()))
return child_range(C, C + 1);
@@ -2401,6 +2409,24 @@ void OMPClausePrinter::VisitOMPGrainsizeClause(OMPGrainsizeClause *Node) {
OS << ")";
}
+void OMPClausePrinter::VisitOMPGraphIdClause(OMPGraphIdClause *Node) {
+ OS << "graph_id";
+ if (Expr *E = Node->getId()) {
+ OS << "(";
+ E->printPretty(OS, nullptr, Policy, 0);
+ OS << ")";
+ }
+}
+
+void OMPClausePrinter::VisitOMPGraphResetClause(OMPGraphResetClause *Node) {
+ OS << "graph_reset";
+ if (Expr *E = Node->getCondition()) {
+ OS << "(";
+ E->printPretty(OS, nullptr, Policy, 0);
+ OS << ")";
+ }
+}
+
void OMPClausePrinter::VisitOMPNumTasksClause(OMPNumTasksClause *Node) {
OS << "num_tasks(";
OpenMPNumTasksClauseModifier Modifier = Node->getModifier();
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index f276647c71cb72..b898ac61226d68 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -958,6 +958,14 @@ void OMPClauseProfiler::VisitOMPGrainsizeClause(const OMPGrainsizeClause *C) {
if (C->getGrainsize())
Profiler->VisitStmt(C->getGrainsize());
}
+void OMPClauseProfiler::VisitOMPGraphIdClause(const OMPGraphIdClause *C) {
+ if (C->getId())
+ Profiler->VisitStmt(C->getId());
+}
+void OMPClauseProfiler::VisitOMPGraphResetClause(const OMPGraphResetClause *C) {
+ if (C->getCondition())
+ Profiler->VisitStmt(C->getCondition());
+}
void OMPClauseProfiler::VisitOMPNumTasksClause(const OMPNumTasksClause *C) {
VisitOMPClauseWithPreInit(C);
if (C->getNumTasks())
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index ca0d3a0f6b84e4..77c383d53e2d30 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -330,6 +330,8 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
case OMPC_when:
case OMPC_append_args:
case OMPC_looprange:
+ case OMPC_graph_id:
+ case OMPC_graph_reset:
break;
default:
break;
@@ -728,6 +730,8 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
case OMPC_when:
case OMPC_append_args:
case OMPC_looprange:
+ case OMPC_graph_id:
+ case OMPC_graph_reset:
break;
default:
break;
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 1e3e400cc7d055..1741af8c7e6053 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3379,6 +3379,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
case OMPC_depth:
case OMPC_align:
case OMPC_message:
+ case OMPC_graph_id:
case OMPC_ompx_dyn_cgroup_mem:
case OMPC_dyn_groupprivate:
case OMPC_transparent:
@@ -3530,6 +3531,20 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
else
Clause = ParseOpenMPClause(CKind, WrongDirective);
break;
+ case OMPC_graph_reset:
+ if (!FirstClause) {
+ Diag(Tok, diag::err_omp_more_one_clause)
+ << getOpenMPDirectiveName(DKind, OMPVersion)
+ << getOpenMPClauseName(CKind) << 0;
+ ErrorFound = true;
+ }
+
+ if (PP.LookAhead(/*N=*/0).is(tok::l_paren)) {
+ Clause = ParseOpenMPSingleExprClause(CKind, WrongDirective);
+ } else {
+ Clause = ParseOpenMPClause(CKind, WrongDirective);
+ }
+ break;
case OMPC_self_maps:
// OpenMP [6.0, self_maps clause]
if (getLangOpts().OpenMP < 60) {
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index e52cae2a056f78..9be064ecc818c7 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -7247,6 +7247,8 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
case OMPC_final:
case OMPC_priority:
case OMPC_novariants:
+ case OMPC_graph_id:
+ case OMPC_graph_reset:
case OMPC_nocontext:
// Do not analyze if no parent parallel directive.
if (isOpenMPParallelDirective(Kind))
@@ -18397,6 +18399,12 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
case OMPC_detach:
Res = ActOnOpenMPDetachClause(Expr, StartLoc, LParenLoc, EndLoc);
break;
+ case OMPC_graph_id:
+ Res = ActOnOpenMPGraphIdClause(Expr, StartLoc, LParenLoc, EndLoc);
+ break;
+ case OMPC_graph_reset:
+ Res = ActOnOpenMPGraphResetClause(Expr, StartLoc, LParenLoc, EndLoc);
+ break;
case OMPC_novariants:
Res = ActOnOpenMPNovariantsClause(Expr, StartLoc, LParenLoc, EndLoc);
break;
@@ -19242,6 +19250,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPSimpleClause(
case OMPC_match:
case OMPC_nontemporal:
case OMPC_destroy:
+ case OMPC_graph_id:
+ case OMPC_graph_reset:
case OMPC_novariants:
case OMPC_nocontext:
case OMPC_detach:
@@ -19994,6 +20004,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprWithArgClause(
case OMPC_severity:
case OMPC_message:
case OMPC_destroy:
+ case OMPC_graph_id:
+ case OMPC_graph_reset:
case OMPC_novariants:
case OMPC_nocontext:
case OMPC_detach:
@@ -20213,6 +20225,10 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
case OMPC_ompx_bare:
Res = ActOnOpenMPXBareClause(StartLoc, EndLoc);
break;
+ case OMPC_graph_reset:
+ Res = ActOnOpenMPGraphResetClause(/*Condition=*/nullptr, StartLoc,
+ SourceLocation(), EndLoc);
+ break;
case OMPC_if:
case OMPC_final:
case OMPC_num_threads:
@@ -20268,6 +20284,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
case OMPC_at:
case OMPC_severity:
case OMPC_message:
+ case OMPC_graph_id:
case OMPC_novariants:
case OMPC_nocontext:
case OMPC_detach:
@@ -20614,6 +20631,47 @@ OMPClause *SemaOpenMP::ActOnOpenMPDestroyClause(Expr *InteropVar,
OMPDestroyClause(InteropVar, StartLoc, LParenLoc, VarLoc, EndLoc);
}
+OMPClause *SemaOpenMP::ActOnOpenMPGraphIdClause(Expr *Id,
+ SourceLocation StartLoc,
+ SourceLocation LParenLoc,
+ SourceLocation EndLoc) {
+ Expr *ValExpr = Id;
+
+ if (!Id->isValueDependent() && !Id->isTypeDependent() &&
+ !Id->isInstantiationDependent() &&
+ !Id->containsUnexpandedParameterPack()) {
+ ExprResult Val = PerformOpenMPImplicitIntegerConversion(LParenLoc, Id);
+ if (Val.isInvalid())
+ return nullptr;
+
+ ValExpr = Val.get();
+ }
+
+ return new (getASTContext())
+ OMPGraphIdClause(ValExpr, StartLoc, LParenLoc, EndLoc);
+}
+
+OMPClause *SemaOpenMP::ActOnOpenMPGraphResetClause(Expr *Condition,
+ SourceLocation StartLoc,
+ SourceLocation LParenLoc,
+ SourceLocation EndLoc) {
+ Expr *ValExpr = Condition;
+ if (Condition && LParenLoc.isValid()) {
+ if (!Condition->isValueDependent() && !Condition->isTypeDependent() &&
+ !Condition->isInstantiationDependent() &&
+ !Condition->containsUnexpandedParameterPack()) {
+ ExprResult Val = SemaRef.CheckBooleanCondition(StartLoc, Condition);
+ if (Val.isInvalid())
+ return nullptr;
+
+ ValExpr = Val.get();
+ }
+ }
+
+ return new (getASTContext())
+ OMPGraphResetClause(ValExpr, StartLoc, LParenLoc, EndLoc);
+}
+
OMPClause *SemaOpenMP::ActOnOpenMPNovariantsClause(Expr *Condition,
SourceLocation StartLoc,
SourceLocation LParenLoc,
@@ -20941,6 +20999,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
case OMPC_severity:
case OMPC_message:
case OMPC_destroy:
+ case OMPC_graph_id:
+ case OMPC_graph_reset:
case OMPC_novariants:
case OMPC_nocontext:
case OMPC_detach:
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 390ed4835e5364..27070b434c8655 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -2206,6 +2206,29 @@ class TreeTransform {
LParenLoc, EndLoc);
}
+ /// Build a new OpenMP 'graph_id' clause.
+ ///
+ /// By default, performs semantic analysis to build the new OpenMP clause.
+ /// Subclasses may override this routine to provide different behavior.
+ OMPClause *RebuildOMPGraphIdClause(Expr *Condition, SourceLocation StartLoc,
+ SourceLocation LParenLoc,
+ SourceLocation EndLoc) {
+ return getSema().OpenMP().ActOnOpenMPGraphIdClause(Condition, StartLoc,
+ LParenLoc, EndLoc);
+ }
+
+ /// Build a new OpenMP 'graph_reset' clause.
+ ///
+ /// By default, performs semantic analysis to build the new OpenMP clause.
+ /// Subclasses may override this routine to provide different behavior.
+ OMPClause *RebuildOMPGraphResetClause(Expr *Condition,
+ SourceLocation StartLoc,
+ SourceLocation LParenLoc,
+ SourceLocation EndLoc) {
+ return getSema().OpenMP().ActOnOpenMPGraphResetClause(Condition, StartLoc,
+ LParenLoc, EndLoc);
+ }
+
/// Build a new OpenMP 'grainsize' clause.
///
/// By default, performs semantic analysis to build the new statement.
@@ -12094,6 +12117,26 @@ TreeTransform<Derived>::TransformOMPPriorityClause(OMPPriorityClause *C) {
E.get(), C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
}
+template <typename Derived>
+OMPClause *
+TreeTransform<Derived>::TransformOMPGraphIdClause(OMPGraphIdClause *C) {
+ ExprResult Cond = getDerived().TransformExpr(C->getId());
+ if (Cond.isInvalid())
+ return nullptr;
+ return getDerived().RebuildOMPGraphIdClause(
+ Cond.get(), C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
+}
+
+template <typename Derived>
+OMPClause *
+TreeTransform<Derived>::TransformOMPGraphResetClause(OMPGraphResetClause *C) {
+ ExprResult Cond = getDerived().TransformExpr(C->getCondition());
+ if (Cond.isInvalid())
+ return nullptr;
+ return getDerived().RebuildOMPGraphResetClause(
+ Cond.get(), C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
+}
+
template <typename Derived>
OMPClause *
TreeTransform<Derived>::TransformOMPGrainsizeClause(OMPGrainsizeClause *C) {
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index cd7d9f24924f82..7f2a811eaf7f24 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11900,6 +11900,12 @@ OMPClause *OMPClauseReader::readClause() {
case llvm::omp::OMPC_grainsize:
C = new (Context) OMPGrainsizeClause();
break;
+ case llvm::omp::OMPC_graph_id:
+ C = new (Context) OMPGraphIdClause();
+ break;
+ case llvm::omp::OMPC_graph_reset:
+ C = new (Context) OMPGraphResetClause();
+ break;
case llvm::omp::OMPC_num_tasks:
C = new (Context) OMPNumTasksClause();
break;
@@ -12842,6 +12848,16 @@ void OMPClauseReader::VisitOMPPriorityClause(OMPPriorityClause *C) {
C->setLParenLoc(Record.readSourceLocation());
}
+void OMPClauseReader::VisitOMPGraphIdClause(OMPGraphIdClause *C) {
+ C->setId(Record.readSubExpr());
+ C->setLParenLoc(Record.readSourceLocation());
+}
+
+void OMPClauseReader::VisitOMPGraphResetClause(OMPGraphResetClause *C) {
+ C->setCondition(Record.readSubExpr());
+ C->setLParenLoc(Record.readSourceLocation());
+}
+
void OMPClauseReader::VisitOMPGrainsizeClause(OMPGrainsizeClause *C) {
VisitOMPClauseWithPreInit(C);
C->setModifier(Record.readEnum<OpenMPGrainsizeClauseModifier>());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index de1c2449138e58..c0b77441e4f290 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8734,6 +8734,16 @@ void OMPClauseWriter::VisitOMPPriorityClause(OMPPriorityClause *C) {
Record.AddSourceLocation(C->getLParenLoc());
}
+void OMPClauseWriter::VisitOMPGraphIdClause(OMPGraphIdClause *C) {
+ Record.AddStmt(C->getId());
+ Record.AddSourceLocation(C->getLParenLoc());
+}
+
+void OMPClauseWriter::VisitOMPGraphResetClause(OMPGraphResetClause *C) {
+ Record.AddStmt(C->getCondition());
+ Record.AddSourceLocation(C->getLParenLoc());
+}
+
void OMPClauseWriter::VisitOMPGrainsizeClause(OMPGrainsizeClause *C) {
VisitOMPClauseWithPreInit(C);
Record.writeEnum(C->getModifier());
diff --git a/clang/test/OpenMP/taskgraph_ast_print.cpp b/clang/test/OpenMP/taskgraph_ast_print.cpp
new file mode 100644
index 00000000000000..bfbe829fc9f317
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_ast_print.cpp
@@ -0,0 +1,31 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+int main() {
+ int x = 0, y = 0;
+
+#pragma omp taskgraph
+// CHECK: #pragma omp taskgraph
+ {
+#pragma omp task depend(in: x) depend(out: y)
+// CHECK: #pragma omp task depend(in : x) depend(out : y)
+ {
+ y = x;
+ }
+#pragma omp task depend(inout: x, y)
+// CHECK: #pragma omp task depend(inout : x,y)
+ {
+ x++;
+ y++;
+ }
+ }
+
+ return 0;
+}
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_clauses_ast_print.cpp b/clang/test/OpenMP/taskgraph_clauses_ast_print.cpp
new file mode 100644
index 00000000000000..8ac721634380ad
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_clauses_ast_print.cpp
@@ -0,0 +1,94 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-dump %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+void taskgraph_clauses() {
+ int A = 1;
+
+ // --- graph_id clause ---
+
+ // DUMP: OMPTaskgraphDirective
+ // DUMP: OMPGraph_idClause
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 0
+ // PRINT: #pragma omp taskgraph graph_id(0)
+ #pragma omp taskgraph graph_id(0)
+ {}
+
+ // DUMP: OMPTaskgraphDirective
+ // DUMP: OMPGraph_idClause
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 42
+ // PRINT: #pragma omp taskgraph graph_id(42)
+ #pragma omp taskgraph graph_id(42)
+ {}
+
+ // DUMP: OMPTaskgraphDirective
+ // DUMP: OMPGraph_idClause
+ // DUMP: BinaryOperator {{.*}} 'int' '+'
+ // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+ // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 10
+ // PRINT: #pragma omp taskgraph graph_id(A + 10)
+ #pragma omp taskgraph graph_id(A + 10)
+ {}
+
+ // --- graph_reset clause ---
+
+ // DUMP: OMPTaskgraphDirective
+ // DUMP: OMPGraph_resetClause
+ // DUMP-NEXT: CXXBoolLiteralExpr {{.*}} 'bool' false
+ // PRINT: #pragma omp taskgraph graph_reset(false)
+ #pragma omp taskgraph graph_reset(false)
+ {}
+
+ // DUMP: OMPTaskgraphDirective
+ // DUMP: OMPGraph_resetClause
+ // DUMP-NEXT: CXXBoolLiteralExpr {{.*}} 'bool' true
+ // PRINT: #pragma omp taskgraph graph_reset(true)
+ #pragma omp taskgraph graph_reset(true)
+ {}
+
+ // DUMP: OMPTaskgraphDirective
+ // DUMP: OMPGraph_resetClause
+ // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+ // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+ // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+ // PRINT: #pragma omp taskgraph graph_reset(A > 5)
+ #pragma omp taskgraph graph_reset(A > 5)
+ {}
+
+ // DUMP: OMPTaskgraphDirective
+ // DUMP: OMPGraph_resetClause
+ // PRINT: #pragma omp taskgraph graph_reset
+ #pragma omp taskgraph graph_reset
+ {}
+
+ // --- Combinations using both clauses ---
+
+ // DUMP: OMPTaskgraphDirective
+ // DUMP: OMPGraph_idClause
+ // DUMP: OMPGraph_resetClause
+ // PRINT: #pragma omp taskgraph graph_id(1) graph_reset(true)
+ #pragma omp taskgraph graph_id(1) graph_reset(true)
+ {}
+
+ // DUMP: OMPTaskgraphDirective
+ // DUMP: OMPGraph_resetClause
+ // DUMP: OMPGraph_idClause
+ // PRINT: #pragma omp taskgraph graph_reset(false) graph_id(2)
+ #pragma omp taskgraph graph_reset(false) graph_id(2)
+ {}
+}
+#endif
diff --git a/clang/test/OpenMP/taskgraph_clauses_messages.cpp b/clang/test/OpenMP/taskgraph_clauses_messages.cpp
new file mode 100644
index 00000000000000..0327f52475d98d
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_clauses_messages.cpp
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=51 -verify=expected,omp51 -fsyntax-only %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -verify=expected,omp60 -fsyntax-only %s
+
+// Tests that the 'graph_id' and 'graph_reset' clauses are accepted in OpenMP 6.0
+// and rejected in prior versions on the 'taskgraph' directive. Also tests
+// duplicate-clause and invalid-condition diagnostics.
+
+void foo() {}
+
+void taskgraph_clauses_messages() {
+ int A = 1;
+
+ // Basic version error tests.
+ #pragma omp taskgraph graph_id(0) // omp51-error {{unexpected OpenMP clause 'graph_id' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+ {}
+
+ #pragma omp taskgraph graph_reset(true) // omp51-error {{unexpected OpenMP clause 'graph_reset' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+ {}
+
+ // Same, without argument.
+ #pragma omp taskgraph graph_reset // omp51-error {{unexpected OpenMP clause 'graph_reset' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+ {}
+
+ // Duplicate clause tests (OMP 6.0 only; in OMP 5.1 both are unexpected).
+ #pragma omp taskgraph graph_id(0) graph_id(1) // omp51-error {{unexpected OpenMP clause 'graph_id' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP clause 'graph_id' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}} expected-error {{directive '#pragma omp taskgraph' cannot contain more than one 'graph_id' clause}}
+ {}
+
+ #pragma omp taskgraph graph_reset(true) graph_reset(false) // omp51-error {{unexpected OpenMP clause 'graph_reset' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP clause 'graph_reset' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}} expected-error {{directive '#pragma omp taskgraph' cannot contain more than one 'graph_reset' clause}}
+ {}
+
+ #pragma omp taskgraph graph_id(foo()) // omp51-error {{unexpected OpenMP clause 'graph_id' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}} omp60-error {{expression must have integral or unscoped enumeration type, not 'void'}}
+ {}
+
+ #pragma omp taskgraph graph_reset(foo()) // omp51-error {{unexpected OpenMP clause 'graph_reset' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}} omp60-error {{value of type 'void' is not contextually convertible to 'bool'}}
+ {}
+}
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 0e2a411f6334c8..1b53d4f2a5d754 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2787,6 +2787,12 @@ void OMPClauseEnqueue::VisitOMPIsDevicePtrClause(
const OMPIsDevicePtrClause *C) {
VisitOMPClauseList(C);
}
+void OMPClauseEnqueue::VisitOMPGraphIdClause(const OMPGraphIdClause *C) {
+ Visitor->AddStmt(C->getId());
+}
+void OMPClauseEnqueue::VisitOMPGraphResetClause(const OMPGraphResetClause *C) {
+ Visitor->AddStmt(C->getCondition());
+}
void OMPClauseEnqueue::VisitOMPHasDeviceAddrClause(
const OMPHasDeviceAddrClause *C) {
VisitOMPClauseList(C);
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 660ab03f9516cd..e6fbee3dd9c1f0 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -256,9 +256,11 @@ def OMPC_GrainSize : Clause<[Spelling<"grainsize">]> {
];
}
def OMPC_GraphId : Clause<[Spelling<"graph_id">]> {
+ let clangClass = "OMPGraphIdClause";
let flangClass = "OmpGraphIdClause";
}
def OMPC_GraphReset : Clause<[Spelling<"graph_reset">]> {
+ let clangClass = "OMPGraphResetClause";
let flangClass = "OmpGraphResetClause";
let isValueOptional = true;
}
>From 635a32996ce31021caddc9a79d27dd7c441fa2df Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Tue, 5 May 2026 08:31:54 -0500
Subject: [PATCH 05/29] [Clang][OpenMP] Support 'taskgraph' semantics via new
libomp API entry points
This patch supports the OpenMP 6.0 'taskgraph' directive for host-based
tasks via several new API entry points for libomp.
New entry points are used for several reasons. The first reason
is to pass extra information from the compiler relevant to the
taskgraph-recording case -- e.g. the __kmpc_taskgraph_task entry point
also takes the task's flags and the sizes of the task and its shared
data, which the runtime needs to copy the task into the recording, and
the task's relocation and clone helpers, which a later patch in this
series fills in (they are null here). The second reason is to
reduce the potential overhead of the taskgraph implementation on the rest
of the runtime. A third intention is to capture OpenMP semantics at a
slightly higher level, to communicate data that is useful for the graph
representation directly in the API call (e.g. for a later patch which adds
target support, dependencies are passed directly to the "omp target" API).
New tests cover codegen additions and restrictions on task-generating
constructs inside taskgraphs, etc.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/194051
---
.../clang/Basic/DiagnosticSemaKinds.td | 4 +
clang/lib/CodeGen/CGOpenMPRuntime.cpp | 731 +++++++++++++-----
clang/lib/CodeGen/CGOpenMPRuntime.h | 4 +-
clang/lib/CodeGen/CodeGenFunction.h | 15 +
clang/lib/Sema/SemaOpenMP.cpp | 39 +
.../test/OpenMP/taskgraph_clauses_codegen.cpp | 102 +++
clang/test/OpenMP/taskgraph_codegen.cpp | 47 ++
.../OpenMP/taskgraph_nesting_messages.cpp | 209 +++++
.../test/OpenMP/taskgraph_taskwait_nodeps.cpp | 12 +
.../include/llvm/Frontend/OpenMP/OMPKinds.def | 11 +
10 files changed, 963 insertions(+), 211 deletions(-)
create mode 100644 clang/test/OpenMP/taskgraph_clauses_codegen.cpp
create mode 100644 clang/test/OpenMP/taskgraph_codegen.cpp
create mode 100644 clang/test/OpenMP/taskgraph_nesting_messages.cpp
create mode 100644 clang/test/OpenMP/taskgraph_taskwait_nodeps.cpp
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 56cc9721781063..82afe9922deb53 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12490,6 +12490,10 @@ def err_omp_single_copyprivate_with_nowait : Error<
"the 'copyprivate' clause must not be used with the 'nowait' clause">;
def err_omp_nowait_clause_without_depend: Error<
"directive '#pragma omp taskwait' cannot use 'nowait' clause without 'depend' clause">;
+def err_omp_taskgraph_taskwait_without_depend: Error<
+ "directive '#pragma omp taskwait' within '#pragma omp taskgraph' must use 'depend' clause to be task-generating">;
+def err_omp_taskgraph_not_task_generating: Error<
+ "'%0' directive is not task-generating and cannot be used within '#pragma omp taskgraph'">;
def note_omp_nowait_clause_here : Note<
"'nowait' clause is here">;
def err_omp_single_decl_in_declare_simd_variant : Error<
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 45eac2df8af3f1..a1dded426de409 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -36,6 +36,7 @@
#include "llvm/ADT/StringExtras.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Frontend/OpenMP/OMP.h.inc"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/GlobalValue.h"
@@ -352,6 +353,26 @@ class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
StringRef HelperName;
};
+/// API for captured statement code generation in OpenMP taskgraphs.
+class CGOpenMPTaskgraphRegionInfo final : public CGOpenMPRegionInfo {
+public:
+ CGOpenMPTaskgraphRegionInfo(const CapturedStmt &CS,
+ const RegionCodeGenTy &CodeGen)
+ : CGOpenMPRegionInfo(CS, TaskgraphOutlinedRegion, CodeGen,
+ llvm::omp::OMPD_taskgraph, false) {}
+
+ const VarDecl *getThreadIDVariable() const override { return 0; }
+
+ /// Get the name of the capture helper.
+ StringRef getHelperName() const override { return "taskgraph.omp_outlined."; }
+
+ static bool classof(const CGCapturedStmtInfo *Info) {
+ return CGOpenMPRegionInfo::classof(Info) &&
+ cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
+ TaskgraphOutlinedRegion;
+ }
+};
+
static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
llvm_unreachable("No codegen for expressions");
}
@@ -2274,24 +2295,110 @@ void CGOpenMPRuntime::emitTaskgraphCall(CodeGenFunction &CGF,
if (!CGF.HaveInsertPoint())
return;
+ // The nogroup clause doesn't support an argument yet. FIXME.
+ const OMPNogroupClause *NoGroupClause = D.getSingleClause<OMPNogroupClause>();
+ llvm::Value *NoGroup;
+ if (NoGroupClause) {
+ NoGroup = CGF.Builder.getInt32(1);
+ } else {
+ NoGroup = CGF.Builder.getInt32(0);
+ }
+
+ const OMPGraphResetClause *GraphResetClause =
+ D.getSingleClause<OMPGraphResetClause>();
+ llvm::Value *GraphReset;
+ if (GraphResetClause) {
+ if (const Expr *Cond = GraphResetClause->getCondition()) {
+ llvm::Value *CondVal = CGF.EvaluateExprAsBool(Cond);
+ GraphReset =
+ CGF.Builder.CreateIntCast(CondVal, CGF.IntTy, /*isSigned=*/false);
+ } else {
+ GraphReset = CGF.Builder.getInt32(1);
+ }
+ } else {
+ GraphReset = CGF.Builder.getInt32(0);
+ }
+
+ llvm::Value *GraphId = llvm::ConstantInt::get(CGM.IntPtrTy, 0);
+ const OMPGraphIdClause *GraphIdClause = D.getSingleClause<OMPGraphIdClause>();
+ if (GraphIdClause) {
+ const auto *E = GraphIdClause->getId();
+ auto *GraphIdVal = CGF.EmitScalarExpr(E);
+ GraphId =
+ CGF.Builder.CreateIntCast(GraphIdVal, CGM.IntPtrTy, /*isSigned=*/false);
+ }
+
CodeGenFunction OutlinedCGF(CGM, /*suppressNewContext=*/true);
const auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
+
+ auto BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
+ CodeGenFunction::OMPWithinTaskgraphRAII WithinTaskgraph(CGF);
+ CGF.EmitStmt(CS->getCapturedStmt());
+ };
+
LValue CapStruct = CGF.InitCapturedStruct(*CS);
+ CGOpenMPTaskgraphRegionInfo TaskgraphRegion(*CS, BodyGen);
+ CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(OutlinedCGF,
+ &TaskgraphRegion);
llvm::Function *OutlinedFn = OutlinedCGF.GenerateCapturedStmtFunction(*CS);
- llvm::Value *CapturedArgsPtr =
+ // Create an internal-linkage global variable to hold the taskgraph handle.
+ std::string GraphHandleName = getName({"omp", "taskgraph", "handle"});
+ auto *GraphHandle = new llvm::GlobalVariable(
+ CGM.getModule(), CGM.VoidPtrTy,
+ /*IsConstant=*/false, llvm::GlobalValue::InternalLinkage,
+ llvm::Constant::getNullValue(CGM.VoidPtrTy), GraphHandleName);
+
+ std::array<llvm::Value *, 8> Args{
+ emitUpdateLocation(CGF, Loc),
+ getThreadID(CGF, Loc),
+ GraphHandle,
+ GraphId,
+ GraphReset,
+ NoGroup,
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(OutlinedFn,
+ CGM.VoidPtrTy),
CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- CapStruct.getPointer(OutlinedCGF), CGM.VoidPtrTy);
+ CapStruct.getPointer(OutlinedCGF), CGM.VoidPtrTy)};
- auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn,
- CapturedArgsPtr);
+ auto &&ThenGen = [&CGF, this, &Args](CodeGenFunction &, PrePostActionTy &) {
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_taskgraph),
+ Args);
};
- RegionCodeGenTy RCG(CodeGen);
- RCG(CGF);
+ auto &&ElseGen = [&CGF, this, &OutlinedFn, &CapStruct, &Loc,
+ &OutlinedCGF](CodeGenFunction &, PrePostActionTy &) {
+ llvm::Value *CapturedArgsPtr =
+ CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ CapStruct.getPointer(OutlinedCGF), CGM.VoidPtrTy);
+
+ auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
+ Action.Enter(CGF);
+ CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn,
+ CapturedArgsPtr);
+ };
+ // A taskgraph region is a taskgroup region whether or not it records: the
+ // if clause governs recording, and nogroup is the only way to drop the
+ // implicit taskgroup. __kmpc_taskgraph supplies one on the recording
+ // path, so emit an explicit one here, on the path that bypasses it.
+ //
+ // FIXME: nogroup should suppress this, but __kmpc_taskgraph ignores the
+ // flag it is handed and creates the taskgroup unconditionally, so
+ // honouring nogroup here alone would make the two paths disagree. Both
+ // need doing together.
+ RegionCodeGenTy RCG(CodeGen);
+ emitTaskgroupRegion(CGF, RCG, Loc);
+ };
+
+ if (IfCond) {
+ emitIfClause(CGF, IfCond, ThenGen, ElseGen);
+ } else {
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_taskgraph),
+ Args);
+ }
}
void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
@@ -3978,11 +4085,11 @@ static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
}
}
-CGOpenMPRuntime::TaskResultTy
-CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
- const OMPExecutableDirective &D,
- llvm::Function *TaskFunction, QualType SharedsTy,
- Address Shareds, const OMPTaskDataTy &Data) {
+CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
+ CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
+ llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
+ const OMPTaskDataTy &Data, bool ForTaskgraph,
+ std::array<llvm::Value *, 3> &TaskAllocArgs) {
ASTContext &C = CGM.getContext();
llvm::SmallVector<PrivateDataTy, 4> Privates;
// Aggregate privates and sort them by the alignment.
@@ -4132,6 +4239,11 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
TaskEntry, KmpRoutineEntryPtrTy)};
llvm::Value *NewTask;
+ if (ForTaskgraph) {
+ TaskAllocArgs[0] = TaskFlags;
+ TaskAllocArgs[1] = KmpTaskTWithPrivatesTySize;
+ TaskAllocArgs[2] = SharedsSize;
+ }
if (D.hasClausesOfKind<OMPNowaitClause>()) {
// Check if we have any device clause associated with the directive.
const Expr *Device = nullptr;
@@ -4890,114 +5002,171 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
if (!CGF.HaveInsertPoint())
return;
- TaskResultTy Result =
- emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
- llvm::Value *NewTask = Result.NewTask;
- llvm::Function *TaskEntry = Result.TaskEntry;
- llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
- LValue TDBase = Result.TDBase;
- const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
- // Process list of dependences.
- Address DependenciesArray = Address::invalid();
- llvm::Value *NumOfElements;
- std::tie(NumOfElements, DependenciesArray) =
- emitDependClause(CGF, Data.Dependences, Loc);
-
- // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
- // libcall.
- // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
- // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
- // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
- // list is not empty
- llvm::Value *ThreadID = getThreadID(CGF, Loc);
- llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
- llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
- llvm::Value *DepTaskArgs[7];
- if (!Data.Dependences.empty()) {
- DepTaskArgs[0] = UpLoc;
- DepTaskArgs[1] = ThreadID;
- DepTaskArgs[2] = NewTask;
- DepTaskArgs[3] = NumOfElements;
- DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
- DepTaskArgs[5] = CGF.Builder.getInt32(0);
- DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
- }
- auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
- &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
- if (!Data.Tied) {
- auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
- LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
- CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
+ auto &&TaskgraphTaskCodeGen = [this, &Loc, &D, TaskFunction, &SharedsTy,
+ &Shareds, &Data](CodeGenFunction &CGF,
+ PrePostActionTy &) {
+ llvm::Value *ThreadId = getThreadID(CGF, Loc);
+ llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+ std::array<llvm::Value *, 10> TGTaskArgs;
+ std::array<llvm::Value *, 3> TaskAllocArgs;
+ TaskResultTy Result = emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy,
+ Shareds, Data, true, TaskAllocArgs);
+ Address DependenciesArray = Address::invalid();
+ llvm::Value *NumOfElements;
+ std::tie(NumOfElements, DependenciesArray) =
+ emitDependClause(CGF, Data.Dependences, Loc);
+ TGTaskArgs[0] = UpLoc;
+ TGTaskArgs[1] = ThreadId;
+ TGTaskArgs[2] = Result.NewTask;
+ TGTaskArgs[3] = TaskAllocArgs[0]; // TaskFlags
+ TGTaskArgs[4] = TaskAllocArgs[1]; // KmpTaskTWithPrivatesTySize
+ TGTaskArgs[5] = TaskAllocArgs[2]; // SharedsSize
+ if (auto RecType = dyn_cast<RecordType>(SharedsTy)) {
+ auto *RD = RecType->getAsRecordDecl();
+ if (RD->fields().empty()) {
+ // FIXME: The condition might not be precisely correct here.
+ TGTaskArgs[5] = CGF.Builder.getSize(0);
+ }
}
- if (!Data.Dependences.empty()) {
- CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
- DepTaskArgs);
+ if (Data.Dependences.size() == 0) {
+ TGTaskArgs[6] = CGF.Builder.getInt32(0);
+ TGTaskArgs[7] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
} else {
- CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_omp_task),
- TaskArgs);
+ TGTaskArgs[6] = NumOfElements;
+ TGTaskArgs[7] = DependenciesArray.emitRawPointer(CGF);
}
- // Check if parent region is untied and build return for untied task;
- if (auto *Region =
- dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
- Region->emitUntiedSwitch(CGF);
+ // The task's relocation and clone helpers: none yet.
+ TGTaskArgs[8] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ TGTaskArgs[9] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_taskgraph_task),
+ TGTaskArgs);
};
- llvm::Value *DepWaitTaskArgs[7];
- if (!Data.Dependences.empty()) {
- DepWaitTaskArgs[0] = UpLoc;
- DepWaitTaskArgs[1] = ThreadID;
- DepWaitTaskArgs[2] = NumOfElements;
- DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
- DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
- DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
- DepWaitTaskArgs[6] =
- llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
- }
- auto &M = CGM.getModule();
- auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
- TaskEntry, &Data, &DepWaitTaskArgs,
- Loc](CodeGenFunction &CGF, PrePostActionTy &) {
- CodeGenFunction::RunCleanupsScope LocalScope(CGF);
- // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
- // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
- // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
- // is specified.
- if (!Data.Dependences.empty())
- CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
- M, OMPRTL___kmpc_omp_taskwait_deps_51),
- DepWaitTaskArgs);
- // Call proxy_task_entry(gtid, new_task);
- auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
- Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
- Action.Enter(CGF);
- llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
- CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
- OutlinedFnArgs);
+ auto &&NonTaskgraphTaskCodeGen = [this, &Loc, &D, TaskFunction, &SharedsTy,
+ &Shareds, IfCond,
+ &Data](CodeGenFunction &CGF,
+ PrePostActionTy &) {
+ std::array<llvm::Value *, 3> DummyArray;
+ TaskResultTy Result = emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy,
+ Shareds, Data, false, DummyArray);
+ llvm::Value *NewTask = Result.NewTask;
+ llvm::Function *TaskEntry = Result.TaskEntry;
+ llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
+ LValue TDBase = Result.TDBase;
+ const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
+ // Process list of dependences.
+ Address DependenciesArray = Address::invalid();
+ llvm::Value *NumOfElements;
+ std::tie(NumOfElements, DependenciesArray) =
+ emitDependClause(CGF, Data.Dependences, Loc);
+
+ // NOTE: routine and part_id fields are initialized by
+ // __kmpc_omp_task_alloc() libcall. Build kmp_int32
+ // __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid, kmp_task_t
+ // *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
+ // ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence list is
+ // not empty
+ llvm::Value *ThreadID = getThreadID(CGF, Loc);
+ llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+ llvm::Value *TaskArgs[] = {UpLoc, ThreadID, NewTask};
+ llvm::Value *DepTaskArgs[7];
+ if (!Data.Dependences.empty()) {
+ DepTaskArgs[0] = UpLoc;
+ DepTaskArgs[1] = ThreadID;
+ DepTaskArgs[2] = NewTask;
+ DepTaskArgs[3] = NumOfElements;
+ DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
+ DepTaskArgs[5] = CGF.Builder.getInt32(0);
+ DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ }
+ auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
+ &DepTaskArgs](CodeGenFunction &CGF,
+ PrePostActionTy &) {
+ if (!Data.Tied) {
+ auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
+ LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
+ CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
+ }
+ if (!Data.Dependences.empty()) {
+ CGF.EmitRuntimeCall(
+ OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
+ DepTaskArgs);
+ } else {
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_omp_task),
+ TaskArgs);
+ }
+ // Check if parent region is untied and build return for untied task;
+ if (auto *Region =
+ dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
+ Region->emitUntiedSwitch(CGF);
};
- // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
- // kmp_task_t *new_task);
- // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
- // kmp_task_t *new_task);
- RegionCodeGenTy RCG(CodeGen);
- CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
- M, OMPRTL___kmpc_omp_task_begin_if0),
- TaskArgs,
- OMPBuilder.getOrCreateRuntimeFunction(
- M, OMPRTL___kmpc_omp_task_complete_if0),
- TaskArgs);
- RCG.setAction(Action);
- RCG(CGF);
+ llvm::Value *DepWaitTaskArgs[7];
+ if (!Data.Dependences.empty()) {
+ DepWaitTaskArgs[0] = UpLoc;
+ DepWaitTaskArgs[1] = ThreadID;
+ DepWaitTaskArgs[2] = NumOfElements;
+ DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
+ DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
+ DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ DepWaitTaskArgs[6] =
+ llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
+ }
+ auto &M = CGM.getModule();
+ auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
+ TaskEntry, &Data, &DepWaitTaskArgs,
+ Loc](CodeGenFunction &CGF, PrePostActionTy &) {
+ CodeGenFunction::RunCleanupsScope LocalScope(CGF);
+ // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
+ // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
+ // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
+ // is specified.
+ if (!Data.Dependences.empty())
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ M, OMPRTL___kmpc_omp_taskwait_deps_51),
+ DepWaitTaskArgs);
+ // Call proxy_task_entry(gtid, new_task);
+ auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
+ Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
+ Action.Enter(CGF);
+ llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
+ CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
+ OutlinedFnArgs);
+ };
+
+ // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
+ // kmp_task_t *new_task);
+ // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
+ // kmp_task_t *new_task);
+ RegionCodeGenTy RCG(CodeGen);
+ CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
+ M, OMPRTL___kmpc_omp_task_begin_if0),
+ TaskArgs,
+ OMPBuilder.getOrCreateRuntimeFunction(
+ M, OMPRTL___kmpc_omp_task_complete_if0),
+ TaskArgs);
+ RCG.setAction(Action);
+ RCG(CGF);
+ };
+
+ if (IfCond) {
+ emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
+ } else {
+ RegionCodeGenTy ThenRCG(ThenCodeGen);
+ ThenRCG(CGF);
+ }
};
- if (IfCond) {
- emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
+ if (CGF.getOMPWithinTaskgraph()) {
+ // Lexically within taskgraph, always replayable.
+ RegionCodeGenTy TaskgraphRCG(TaskgraphTaskCodeGen);
+ TaskgraphRCG(CGF);
} else {
- RegionCodeGenTy ThenRCG(ThenCodeGen);
- ThenRCG(CGF);
+ RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskCodeGen);
+ NonTaskgraphRCG(CGF);
}
}
@@ -5009,15 +5178,11 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
const OMPTaskDataTy &Data) {
if (!CGF.HaveInsertPoint())
return;
- TaskResultTy Result =
- emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
- // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
- // libcall.
- // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
- // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
- // sched, kmp_uint64 grainsize, void *task_dup);
- llvm::Value *ThreadID = getThreadID(CGF, Loc);
- llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+
+ std::array<llvm::Value *, 3> TaskAllocArgs;
+ TaskResultTy TaskInitResult = emitTaskInit(
+ CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data, true, TaskAllocArgs);
+
llvm::Value *IfVal;
if (IfCond) {
IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
@@ -5026,68 +5191,176 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
}
- LValue LBLVal = CGF.EmitLValueForField(
- Result.TDBase,
- *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
- const auto *LBVar =
- cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
- CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
- /*IsInitializer=*/true);
- LValue UBLVal = CGF.EmitLValueForField(
- Result.TDBase,
- *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
- const auto *UBVar =
- cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
- CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
- /*IsInitializer=*/true);
- LValue StLVal = CGF.EmitLValueForField(
- Result.TDBase,
- *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
- const auto *StVar =
- cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
- CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
- /*IsInitializer=*/true);
- // Store reductions address.
- LValue RedLVal = CGF.EmitLValueForField(
- Result.TDBase,
- *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
- if (Data.Reductions) {
- CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
+ enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
+
+ auto &&TaskgraphTaskloopCodeGen = [this, &Loc, &D, &TaskInitResult, IfVal,
+ &Data,
+ &TaskAllocArgs](CodeGenFunction &CGF,
+ PrePostActionTy &) {
+ llvm::Value *ThreadId = getThreadID(CGF, Loc);
+ llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+ std::array<llvm::Value *, 14> TGTaskLoopArgs;
+
+ // This is all copy/pasted from below. Refactor!
+ LValue LBLVal = CGF.EmitLValueForField(
+ TaskInitResult.TDBase,
+ *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+ KmpTaskTLowerBound));
+ const auto *LBVar =
+ cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
+ CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(),
+ LBLVal.getQuals(),
+ /*IsInitializer=*/true);
+ LValue UBLVal = CGF.EmitLValueForField(
+ TaskInitResult.TDBase,
+ *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+ KmpTaskTUpperBound));
+ const auto *UBVar =
+ cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
+ CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(),
+ UBLVal.getQuals(),
+ /*IsInitializer=*/true);
+ LValue StLVal = CGF.EmitLValueForField(
+ TaskInitResult.TDBase,
+ *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+ KmpTaskTStride));
+ const auto *StVar =
+ cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
+ CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(),
+ StLVal.getQuals(), /*IsInitializer=*/true);
+ // Store reductions address.
+ LValue RedLVal = CGF.EmitLValueForField(
+ TaskInitResult.TDBase,
+ *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+ KmpTaskTReductions));
+ if (Data.Reductions) {
+ CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
+ } else {
+ CGF.EmitNullInitialization(RedLVal.getAddress(),
+ CGF.getContext().VoidPtrTy);
+ }
+
+ TGTaskLoopArgs[0] = UpLoc;
+ TGTaskLoopArgs[1] = ThreadId;
+ TGTaskLoopArgs[2] = TaskInitResult.NewTask;
+ TGTaskLoopArgs[3] = TaskAllocArgs[0]; // TaskFlags
+ TGTaskLoopArgs[4] = IfVal;
+ TGTaskLoopArgs[5] = LBLVal.getPointer(CGF);
+ TGTaskLoopArgs[6] = UBLVal.getPointer(CGF);
+ TGTaskLoopArgs[7] = CGF.EmitLoadOfScalar(StLVal, Loc);
+ TGTaskLoopArgs[8] =
+ llvm::ConstantInt::getSigned(CGF.IntTy, Data.Nogroup ? 1 : 0);
+ TGTaskLoopArgs[9] = llvm::ConstantInt::getSigned(
+ CGF.IntTy, Data.Schedule.getPointer()
+ ? Data.Schedule.getInt() ? NumTasks : Grainsize
+ : NoSchedule);
+ TGTaskLoopArgs[10] =
+ Data.Schedule.getPointer()
+ ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
+ /*isSigned=*/false)
+ : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0);
+ TGTaskLoopArgs[11] =
+ llvm::ConstantInt::getSigned(CGF.IntTy, Data.HasModifier ? 1 : 0);
+ TGTaskLoopArgs[12] = TaskInitResult.TaskDupFn
+ ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ TaskInitResult.TaskDupFn, CGF.VoidPtrTy)
+ : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ // The task's relocation helper: none yet.
+ TGTaskLoopArgs[13] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_taskgraph_taskloop),
+ TGTaskLoopArgs);
+ };
+
+ auto &&NonTaskgraphTaskloopCodeGen = [this, &Loc, &D, &TaskInitResult, IfVal,
+ &Data](CodeGenFunction &CGF,
+ PrePostActionTy &) {
+ // NOTE: routine and part_id fields are initialized by
+ // __kmpc_omp_task_alloc() libcall. Call to void __kmpc_taskloop(ident_t
+ // *loc, int gtid, kmp_task_t *task, int if_val, kmp_uint64 *lb, kmp_uint64
+ // *ub, kmp_int64 st, int nogroup, int sched, kmp_uint64 grainsize, void
+ // *task_dup);
+ llvm::Value *ThreadID = getThreadID(CGF, Loc);
+ llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+
+ LValue LBLVal = CGF.EmitLValueForField(
+ TaskInitResult.TDBase,
+ *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+ KmpTaskTLowerBound));
+ const auto *LBVar =
+ cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
+ CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(),
+ LBLVal.getQuals(),
+ /*IsInitializer=*/true);
+ LValue UBLVal = CGF.EmitLValueForField(
+ TaskInitResult.TDBase,
+ *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+ KmpTaskTUpperBound));
+ const auto *UBVar =
+ cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
+ CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(),
+ UBLVal.getQuals(),
+ /*IsInitializer=*/true);
+ LValue StLVal = CGF.EmitLValueForField(
+ TaskInitResult.TDBase,
+ *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+ KmpTaskTStride));
+ const auto *StVar =
+ cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
+ CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(),
+ StLVal.getQuals(),
+ /*IsInitializer=*/true);
+ // Store reductions address.
+ LValue RedLVal = CGF.EmitLValueForField(
+ TaskInitResult.TDBase,
+ *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+ KmpTaskTReductions));
+ if (Data.Reductions) {
+ CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
+ } else {
+ CGF.EmitNullInitialization(RedLVal.getAddress(),
+ CGF.getContext().VoidPtrTy);
+ }
+ llvm::SmallVector<llvm::Value *, 12> TaskArgs{
+ UpLoc,
+ ThreadID,
+ TaskInitResult.NewTask,
+ IfVal,
+ LBLVal.getPointer(CGF),
+ UBLVal.getPointer(CGF),
+ CGF.EmitLoadOfScalar(StLVal, Loc),
+ llvm::ConstantInt::getSigned(
+ CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
+ llvm::ConstantInt::getSigned(
+ CGF.IntTy, Data.Schedule.getPointer()
+ ? Data.Schedule.getInt() ? NumTasks : Grainsize
+ : NoSchedule),
+ Data.Schedule.getPointer()
+ ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
+ /*isSigned=*/false)
+ : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0)};
+ if (Data.HasModifier)
+ TaskArgs.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 1));
+
+ TaskArgs.push_back(TaskInitResult.TaskDupFn
+ ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ TaskInitResult.TaskDupFn, CGF.VoidPtrTy)
+ : llvm::ConstantPointerNull::get(CGF.VoidPtrTy));
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), Data.HasModifier
+ ? OMPRTL___kmpc_taskloop_5
+ : OMPRTL___kmpc_taskloop),
+ TaskArgs);
+ };
+
+ if (CGF.getOMPWithinTaskgraph()) {
+ // Lexically within taskgraph, always replayable.
+ RegionCodeGenTy TaskgraphRCG(TaskgraphTaskloopCodeGen);
+ TaskgraphRCG(CGF);
} else {
- CGF.EmitNullInitialization(RedLVal.getAddress(),
- CGF.getContext().VoidPtrTy);
+ RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskloopCodeGen);
+ NonTaskgraphRCG(CGF);
}
- enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
- llvm::SmallVector<llvm::Value *, 12> TaskArgs{
- UpLoc,
- ThreadID,
- Result.NewTask,
- IfVal,
- LBLVal.getPointer(CGF),
- UBLVal.getPointer(CGF),
- CGF.EmitLoadOfScalar(StLVal, Loc),
- llvm::ConstantInt::getSigned(
- CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
- llvm::ConstantInt::getSigned(
- CGF.IntTy, Data.Schedule.getPointer()
- ? Data.Schedule.getInt() ? NumTasks : Grainsize
- : NoSchedule),
- Data.Schedule.getPointer()
- ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
- /*isSigned=*/false)
- : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0)};
- if (Data.HasModifier)
- TaskArgs.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 1));
-
- TaskArgs.push_back(Result.TaskDupFn
- ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
- Result.TaskDupFn, CGF.VoidPtrTy)
- : llvm::ConstantPointerNull::get(CGF.VoidPtrTy));
- CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), Data.HasModifier
- ? OMPRTL___kmpc_taskloop_5
- : OMPRTL___kmpc_taskloop),
- TaskArgs);
}
/// Emit reduction operation for each element of array (required for
@@ -6229,9 +6502,15 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
CGM.VoidPtrTy)};
- return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_taskred_init),
- Args);
+ if (CGF.getOMPWithinTaskgraph())
+ return CGF.EmitRuntimeCall(
+ OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_taskgraph_taskred_init),
+ Args);
+ else
+ return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_taskred_init),
+ Args);
}
void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
@@ -6305,36 +6584,68 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
llvm::Value *NumOfElements;
std::tie(NumOfElements, DependenciesArray) =
emitDependClause(CGF, Data.Dependences, Loc);
- if (!Data.Dependences.empty()) {
- llvm::Value *DepWaitTaskArgs[7];
- DepWaitTaskArgs[0] = UpLoc;
- DepWaitTaskArgs[1] = ThreadID;
- DepWaitTaskArgs[2] = NumOfElements;
- DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
- DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
- DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
- DepWaitTaskArgs[6] =
- llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
-
- CodeGenFunction::RunCleanupsScope LocalScope(CGF);
- // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,
- // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
- // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,
- // kmp_int32 has_no_wait); if dependence info is specified.
- CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
- M, OMPRTL___kmpc_omp_taskwait_deps_51),
- DepWaitTaskArgs);
+ auto &&TaskgraphTaskwaitCodeGen =
+ [this, UpLoc, ThreadID, NumOfElements, &DependenciesArray,
+ &Data](CodeGenFunction &CGF, PrePostActionTy &) {
+ llvm::Value *TGTaskWaitArgs[5];
+ TGTaskWaitArgs[0] = UpLoc;
+ TGTaskWaitArgs[1] = ThreadID;
+ if (Data.Dependences.empty()) {
+ TGTaskWaitArgs[2] = CGF.Builder.getInt32(0);
+ TGTaskWaitArgs[3] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ } else {
+ TGTaskWaitArgs[2] = NumOfElements;
+ TGTaskWaitArgs[3] = DependenciesArray.emitRawPointer(CGF);
+ }
+ TGTaskWaitArgs[4] =
+ llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
+ CGF.EmitRuntimeCall(
+ OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_taskgraph_taskwait),
+ TGTaskWaitArgs);
+ };
+ auto &&NonTaskgraphTaskwaitCodeGen =
+ [this, UpLoc, ThreadID, NumOfElements, &DependenciesArray, &M,
+ &Data](CodeGenFunction &CGF, PrePostActionTy &) {
+ if (!Data.Dependences.empty()) {
+ llvm::Value *DepWaitTaskArgs[7];
+ DepWaitTaskArgs[0] = UpLoc;
+ DepWaitTaskArgs[1] = ThreadID;
+ DepWaitTaskArgs[2] = NumOfElements;
+ DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
+ DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
+ DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ DepWaitTaskArgs[6] =
+ llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
+
+ CodeGenFunction::RunCleanupsScope LocalScope(CGF);
+
+ // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,
+ // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
+ // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,
+ // kmp_int32 has_no_wait); if dependence info is specified.
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ M, OMPRTL___kmpc_omp_taskwait_deps_51),
+ DepWaitTaskArgs);
+ } else {
+ // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
+ // global_tid);
+ llvm::Value *Args[] = {UpLoc, ThreadID};
+ // Ignore return result until untied tasks are supported.
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ M, OMPRTL___kmpc_omp_taskwait),
+ Args);
+ }
+ };
+ if (CGF.getOMPWithinTaskgraph()) {
+ // Lexically within taskgraph, always replayable.
+ RegionCodeGenTy TaskgraphRCG(TaskgraphTaskwaitCodeGen);
+ TaskgraphRCG(CGF);
} else {
-
- // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
- // global_tid);
- llvm::Value *Args[] = {UpLoc, ThreadID};
- // Ignore return result until untied tasks are supported.
- CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),
- Args);
+ RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskwaitCodeGen);
+ NonTaskgraphRCG(CGF);
}
}
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 8d7094afa702f1..9dd6b3b02f7fdf 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -582,7 +582,9 @@ class CGOpenMPRuntime {
TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
const OMPExecutableDirective &D,
llvm::Function *TaskFunction, QualType SharedsTy,
- Address Shareds, const OMPTaskDataTy &Data);
+ Address Shareds, const OMPTaskDataTy &Data,
+ bool ForTaskgraph,
+ std::array<llvm::Value *, 3> &TaskAllocArgs);
/// Emit update for lastprivate conditional data.
void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index e972d9b1cc1b32..85d518f28c4f38 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -851,6 +851,21 @@ class CodeGenFunction : public CodeGenTypeCache {
}
};
+ bool OMPWithinTaskgraph = false;
+
+ bool getOMPWithinTaskgraph() { return OMPWithinTaskgraph; }
+ void setOMPWithinTaskgraph(bool In) { OMPWithinTaskgraph = In; }
+
+ class OMPWithinTaskgraphRAII {
+ CodeGenFunction &CGF;
+
+ public:
+ OMPWithinTaskgraphRAII(CodeGenFunction &CGF_) : CGF(CGF_) {
+ CGF.setOMPWithinTaskgraph(true);
+ }
+ ~OMPWithinTaskgraphRAII() { CGF.setOMPWithinTaskgraph(false); }
+ };
+
template <class T>
typename DominatingValue<T>::saved_type saveValueInCond(T value) {
return DominatingValue<T>::save(*this, value);
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 9be064ecc818c7..58078a6a43a577 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -5402,6 +5402,24 @@ static bool checkCancelRegion(Sema &SemaRef, OpenMPDirectiveKind CurrentRegion,
return true;
}
+// Return TRUE if KIND is a task-generating construct. In the case of compound
+// directives, the outermost leaf kind is tested.
+static bool isOpenMPExplicitTaskGeneratingDirective(OpenMPDirectiveKind Kind) {
+ switch (getLeafConstructsOrSelf(Kind).front()) {
+ case OMPD_target:
+ case OMPD_target_data:
+ case OMPD_target_enter_data:
+ case OMPD_target_exit_data:
+ case OMPD_target_update:
+ case OMPD_task:
+ case OMPD_taskloop:
+ case OMPD_taskwait:
+ return true;
+ default:
+ return false;
+ }
+}
+
static bool checkNestingOfRegions(Sema &SemaRef, const DSAStackTy *Stack,
OpenMPDirectiveKind CurrentRegion,
const DeclarationNameInfo &CurrentName,
@@ -5474,6 +5492,23 @@ static bool checkNestingOfRegions(Sema &SemaRef, const DSAStackTy *Stack,
SemaRef.Diag(StartLoc, diag::err_omp_prohibited_region_atomic);
return true;
}
+ if (ParentRegion == OMPD_taskgraph &&
+ getDirectiveCategory(CurrentRegion) == Category::Executable &&
+ !isOpenMPExplicitTaskGeneratingDirective(CurrentRegion)) {
+ // OpenMP 6.0 [14.3, taskgraph Construct, Restrictions]
+ // Task-generating constructs are the only constructs that may be
+ // encountered as part of the taskgraph region. A construct that is not
+ // task-generating generates no node to record, so it would run on the
+ // recording execution and then be absent from every replay. The parent
+ // directive is the whole test: [2, region] puts neither the body of a
+ // generated task nor a target region in the encountering region, so a
+ // construct below one of those is not in the taskgraph region and is not
+ // restricted. Only constructs are restricted, hence the category test,
+ // which leaves the informational and utility directives alone.
+ SemaRef.Diag(StartLoc, diag::err_omp_taskgraph_not_task_generating)
+ << getOpenMPDirectiveName(CurrentRegion, OMPVersion);
+ return true;
+ }
if (CurrentRegion == OMPD_section) {
// OpenMP [2.7.2, sections Construct, Restrictions]
// Orphaned section directives are prohibited. That is, the section
@@ -12404,6 +12439,10 @@ SemaOpenMP::ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
Diag(StartLoc, diag::err_omp_nowait_clause_without_depend);
return StmtError();
}
+ if (DSAStack->getParentDirective() == OMPD_taskgraph && !HasDependC) {
+ Diag(StartLoc, diag::err_omp_taskgraph_taskwait_without_depend);
+ return StmtError();
+ }
return OMPTaskwaitDirective::Create(getASTContext(), StartLoc, EndLoc,
Clauses);
diff --git a/clang/test/OpenMP/taskgraph_clauses_codegen.cpp b/clang/test/OpenMP/taskgraph_clauses_codegen.cpp
new file mode 100644
index 00000000000000..631549c511055b
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_clauses_codegen.cpp
@@ -0,0 +1,102 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+// expected-no-diagnostics
+
+// Codegen for the graph_id, graph_reset and if clauses. For graph_id and
+// graph_reset the arguments of interest are the 4th (graph_id, i64) and 5th
+// (graph_reset, i32) of __kmpc_taskgraph.
+
+#ifndef HEADER
+#define HEADER
+
+void body();
+
+// CHECK-LABEL: define {{.*}} @_Z10no_clausesv
+void no_clauses() {
+ // CHECK: call void @__kmpc_taskgraph(ptr {{.*}}, i32 %{{.*}}, ptr @.omp.taskgraph.handle{{[^,]*}}, i64 0, i32 0, i32 0,
+#pragma omp taskgraph
+ { body(); }
+}
+
+// An omitted graph_reset argument has the optional property: it means "reset",
+// so it must lower to a constant 1 -- not to 0, and not to a null-pointer
+// dereference in the frontend.
+// CHECK-LABEL: define {{.*}} @_Z17reset_no_argumentv
+void reset_no_argument() {
+ // CHECK: call void @__kmpc_taskgraph(ptr {{.*}}, i32 %{{.*}}, ptr @.omp.taskgraph.handle{{[^,]*}}, i64 0, i32 1, i32 0,
+#pragma omp taskgraph graph_reset
+ { body(); }
+}
+
+// A true condition must agree with the omitted form rather than differing by a
+// sign extension (it used to pass -1).
+// CHECK-LABEL: define {{.*}} @_Z10reset_truev
+void reset_true() {
+ // CHECK: call void @__kmpc_taskgraph(ptr {{.*}}, i32 %{{.*}}, ptr @.omp.taskgraph.handle{{[^,]*}}, i64 0, i32 1, i32 0,
+#pragma omp taskgraph graph_reset(true)
+ { body(); }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z11reset_falsev
+void reset_false() {
+ // CHECK: call void @__kmpc_taskgraph(ptr {{.*}}, i32 %{{.*}}, ptr @.omp.taskgraph.handle{{[^,]*}}, i64 0, i32 0, i32 0,
+#pragma omp taskgraph graph_reset(false)
+ { body(); }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z10reset_exprb
+void reset_expr(bool c) {
+ // CHECK: [[TOBOOL:%.*]] = icmp ne i8 %{{.*}}, 0
+ // CHECK-NEXT: [[RESET:%.*]] = zext i1 [[TOBOOL]] to i32
+ // CHECK-NEXT: call void @__kmpc_taskgraph(ptr {{.*}}, i32 %{{.*}}, ptr @.omp.taskgraph.handle{{[^,]*}}, i64 0, i32 [[RESET]], i32 0,
+#pragma omp taskgraph graph_reset(c)
+ { body(); }
+}
+
+// An omitted argument combines with graph_id as usual.
+// CHECK-LABEL: define {{.*}} @_Z24id_and_reset_no_argumenti
+void id_and_reset_no_argument(int id) {
+ // CHECK: [[GID:%.*]] = zext i32 %{{.*}} to i64
+ // CHECK-NEXT: call void @__kmpc_taskgraph(ptr {{.*}}, i32 %{{.*}}, ptr @.omp.taskgraph.handle{{[^,]*}}, i64 [[GID]], i32 1, i32 0,
+#pragma omp taskgraph graph_id(id) graph_reset
+ { body(); }
+}
+
+// The if clause selects whether the region is recorded, and nothing else. In
+// particular the region is a taskgroup region either way: __kmpc_taskgraph
+// supplies the implicit taskgroup on the recording path, so the path that
+// bypasses it has to emit one.
+
+// CHECK-LABEL: define {{.*}} @_Z8if_falsev
+void if_false() {
+ // CHECK-NOT: @__kmpc_taskgraph(
+ // CHECK: call void @__kmpc_taskgroup(
+ // CHECK-NEXT: call void @taskgraph.omp_outlined.
+ // CHECK-NEXT: call void @__kmpc_end_taskgroup(
+#pragma omp taskgraph if(false)
+ { body(); }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z7if_truev
+void if_true() {
+ // CHECK-NOT: @__kmpc_taskgroup(
+ // CHECK: call void @__kmpc_taskgraph(
+#pragma omp taskgraph if(true)
+ { body(); }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z6if_dynb
+void if_dyn(bool c) {
+ // CHECK: br i1 %{{.*}}, label %[[THEN:.+]], label %[[ELSE:.+]]
+ // CHECK: [[THEN]]:
+ // CHECK-NEXT: call void @__kmpc_taskgraph(
+ // CHECK: [[ELSE]]:
+ // CHECK-NEXT: call void @__kmpc_taskgroup(
+ // CHECK-NEXT: call void @taskgraph.omp_outlined.
+ // CHECK-NEXT: call void @__kmpc_end_taskgroup(
+#pragma omp taskgraph if(taskgraph: c)
+ { body(); }
+}
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_codegen.cpp b/clang/test/OpenMP/taskgraph_codegen.cpp
new file mode 100644
index 00000000000000..e6e718c2cd6d5f
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_codegen.cpp
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --functions "main" --prefix-filecheck-ir-name _
+
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// CHECK-LABEL: @main(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[X:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[Y:%.*]] = alloca i32, align 4
+// CHECK-NEXT: [[AGG_CAPTURED:%.*]] = alloca [[STRUCT_ANON:%.*]], align 8
+// CHECK-NEXT: [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1:[0-9]+]])
+// CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4
+// CHECK-NEXT: store i32 0, ptr [[X]], align 4
+// CHECK-NEXT: store i32 0, ptr [[Y]], align 4
+// CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON]], ptr [[AGG_CAPTURED]], i32 0, i32 0
+// CHECK-NEXT: store ptr [[X]], ptr [[TMP1]], align 8
+// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON]], ptr [[AGG_CAPTURED]], i32 0, i32 1
+// CHECK-NEXT: store ptr [[Y]], ptr [[TMP2]], align 8
+// CHECK-NEXT: call void @__kmpc_taskgraph(ptr @[[GLOB1]], i32 [[TMP0]], ptr @.omp.taskgraph.handle, i64 0, i32 0, i32 0, ptr @taskgraph.omp_outlined., ptr [[AGG_CAPTURED]])
+// CHECK-NEXT: ret i32 0
+//
+int main() {
+ int x = 0, y = 0;
+
+#pragma omp taskgraph
+ {
+#pragma omp task depend(in: x) depend(out: y)
+ {
+ y = x;
+ }
+#pragma omp task depend(inout: x, y)
+ {
+ x++;
+ y++;
+ }
+ }
+
+ return 0;
+}
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_nesting_messages.cpp b/clang/test/OpenMP/taskgraph_nesting_messages.cpp
new file mode 100644
index 00000000000000..942a7a34457686
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_nesting_messages.cpp
@@ -0,0 +1,209 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -verify -fsyntax-only %s
+
+// OpenMP 6.0 [14.3, taskgraph Construct, Restrictions]: task-generating
+// constructs are the only constructs that may be encountered as part of the
+// taskgraph region. A construct that is not task-generating records no node,
+// so it would run on the recording execution and then be missing from every
+// replay. The 6.1 draft keeps the restriction and spells it with the sharper
+// explicit-task-generating property. The same cases are covered for flang in
+// flang/test/Semantics/OpenMP/taskgraph.f90.
+
+void not_task_generating(int n) {
+#pragma omp taskgraph
+ {
+ // A taskgraph is itself not task-generating, so it cannot be nested.
+#pragma omp taskgraph // expected-error {{'taskgraph' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ {}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp parallel // expected-error {{'parallel' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ {}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp taskgroup // expected-error {{'taskgroup' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ {}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp critical // expected-error {{'critical' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ {}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp barrier // expected-error {{'barrier' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp taskyield // expected-error {{'taskyield' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp flush // expected-error {{'flush' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp single // expected-error {{'single' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ {}
+ }
+
+#pragma omp taskgraph
+ {
+ int x = 0;
+#pragma omp atomic // expected-error {{'atomic' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ x++;
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp unroll partial(2) // expected-error {{'unroll' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ for (int i = 0; i < n; ++i)
+ ;
+ }
+}
+
+// Only the outermost leaf of a compound construct decides. It is the leaf that
+// generates the explicit task; the remaining leaves are encountered inside that
+// task's region, which [2, region] excludes from the taskgraph region.
+void compound(int n) {
+#pragma omp taskgraph
+ {
+#pragma omp masked taskloop // expected-error {{'masked taskloop' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ for (int i = 0; i < n; ++i)
+ ;
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp parallel masked taskloop // expected-error {{'parallel masked taskloop' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ for (int i = 0; i < n; ++i)
+ ;
+ }
+
+ // Ok: the target leaf generates the target task.
+#pragma omp taskgraph
+ {
+#pragma omp target teams distribute parallel for
+ for (int i = 0; i < n; ++i)
+ ;
+#pragma omp target parallel
+ {}
+#pragma omp taskloop simd
+ for (int i = 0; i < n; ++i)
+ ;
+ }
+}
+
+// The task-generating constructs, which are also the ones the replayable clause
+// may appear on, plus target data: a composite of target_enter_data, a task and
+// target_exit_data, so task-generating as well.
+void task_generating(int n) {
+ int x = 0;
+
+#pragma omp taskgraph
+ {
+#pragma omp task
+ {}
+#pragma omp taskloop
+ for (int i = 0; i < n; ++i)
+ ;
+#pragma omp target
+ {}
+#pragma omp target enter data map(to : x)
+#pragma omp target update to(x)
+#pragma omp target exit data map(from : x)
+#pragma omp target data map(tofrom : x)
+ {}
+#pragma omp taskwait depend(in : x)
+ }
+}
+
+// A taskwait is task-generating only with a depend clause, which has its own
+// diagnostic rather than this one.
+void taskwait_without_depend() {
+#pragma omp taskgraph
+ {
+#pragma omp taskwait // expected-error {{directive '#pragma omp taskwait' within '#pragma omp taskgraph' must use 'depend' clause to be task-generating}}
+ }
+}
+
+// Only constructs are restricted, so the informational and utility directives
+// are left alone.
+void not_a_construct() {
+#pragma omp taskgraph
+ {
+#pragma omp nothing
+#pragma omp assume no_openmp_routines
+ {}
+ }
+}
+
+// [2, region] puts neither the body of a generated task nor a target region in
+// the region of the encountering thread, so a construct below one of those is
+// not encountered in the taskgraph region.
+void below_a_generated_task(int n) {
+#pragma omp taskgraph
+ {
+#pragma omp task
+ {
+#pragma omp parallel
+ {}
+#pragma omp taskgraph
+ {}
+ }
+
+#pragma omp taskloop
+ for (int i = 0; i < n; ++i) {
+#pragma omp critical
+ {}
+ }
+
+#pragma omp target
+ {
+#pragma omp parallel
+ {}
+ }
+
+ // The body of a target data region is that of its constituent task.
+#pragma omp target data map(tofrom : n)
+ {
+#pragma omp taskgroup
+ {}
+ }
+ }
+}
+
+// Outside a taskgraph region none of this applies.
+void outside() {
+#pragma omp parallel
+ {
+#pragma omp taskgroup
+ {
+#pragma omp taskgraph
+ {
+#pragma omp task
+ {}
+ }
+ }
+ }
+}
+
+template <typename T> void templated(T n) {
+#pragma omp taskgraph
+ {
+#pragma omp parallel // expected-error {{'parallel' directive is not task-generating and cannot be used within '#pragma omp taskgraph'}}
+ {}
+#pragma omp task
+ {}
+ }
+}
+
+void instantiate() { templated<int>(4); }
diff --git a/clang/test/OpenMP/taskgraph_taskwait_nodeps.cpp b/clang/test/OpenMP/taskgraph_taskwait_nodeps.cpp
new file mode 100644
index 00000000000000..9f088ed523f26c
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_taskwait_nodeps.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -verify=expected -fsyntax-only %s
+
+int main() {
+#pragma omp taskgraph
+ {
+#pragma omp taskwait // expected-error {{directive '#pragma omp taskwait' within '#pragma omp taskgraph' must use 'depend' clause to be task-generating}}
+ }
+
+#pragma omp taskwait
+
+ return 0;
+}
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 6dc100e2592cd7..1d41c41aad3e68 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -366,6 +366,17 @@ __OMP_RTL(__kmpc_omp_task, false, Int32, IdentPtr, Int32,
/* kmp_task_t */ VoidPtr)
__OMP_RTL(__kmpc_end_taskgroup, false, Void, IdentPtr, Int32)
__OMP_RTL(__kmpc_taskgroup, false, Void, IdentPtr, Int32)
+__OMP_RTL(__kmpc_taskgraph, false, Void, IdentPtr, Int32, VoidPtrPtr, SizeTy,
+ Int32, Int32, VoidPtr, VoidPtr)
+__OMP_RTL(__kmpc_taskgraph_task, false, Int32, IdentPtr, Int32, VoidPtr, Int32,
+ SizeTy, SizeTy, Int32, VoidPtr, VoidPtr, VoidPtr)
+__OMP_RTL(__kmpc_taskgraph_taskloop, false, Int32, IdentPtr, Int32, VoidPtr,
+ Int32, Int32, Int64Ptr, Int64Ptr, Int64,
+ Int32, Int32, Int64, Int32, VoidPtr, VoidPtr)
+__OMP_RTL(__kmpc_taskgraph_taskwait, false, Void, IdentPtr, Int32, Int32,
+ VoidPtr, Int32)
+__OMP_RTL(__kmpc_taskgraph_taskred_init, false, /* kmp_taskgroup */ VoidPtr,
+ Int32, Int32, VoidPtr)
__OMP_RTL(__kmpc_omp_task_begin_if0, false, Void, IdentPtr, Int32,
/* kmp_task_t */ VoidPtr)
__OMP_RTL(__kmpc_omp_task_complete_if0, false, Void, IdentPtr, Int32,
>From d61a4c56796b6684a463846ed29821ef8d054606 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Mon, 27 Apr 2026 13:55:47 -0500
Subject: [PATCH 06/29] [Clang][OpenMP] Add 'replayable' clause (for
'taskgraph' support)
This patch adds support for the 'replayable' clause to Clang. This is
used to indicate whether the directive the clause is used on participates
in taskgraph recording and replay: thus, when the clause is present (and
its argument is true), the new API routines introduced in a previous
patch are emitted, even when the directive is not lexically nested
within a taskgraph construct. Conversely, a construct inside a taskgraph
region is replayable unless its 'replayable' clause says otherwise, so
'replayable(false)' there keeps it out of the recording. The clause is
accepted on the target-family compound directives as well ('target teams',
'target parallel for' and the rest), as OpenMP 6.0 allows.
Tests have been added to cover parsing and diagnostics, including
interaction with reductions.
Assisted-By: Codex
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/194053
---
clang/include/clang/AST/OpenMPClause.h | 69 +++++
clang/include/clang/AST/RecursiveASTVisitor.h | 7 +
.../clang/Basic/DiagnosticSemaKinds.td | 2 +
clang/include/clang/Sema/SemaOpenMP.h | 6 +
clang/lib/AST/OpenMPClause.cpp | 15 +
clang/lib/AST/StmtProfile.cpp | 5 +
clang/lib/CodeGen/CGOpenMPRuntime.cpp | 84 ++---
clang/lib/CodeGen/CGOpenMPRuntime.h | 9 +-
clang/lib/CodeGen/CGStmtOpenMP.cpp | 77 ++++-
clang/lib/Parse/ParseOpenMP.cpp | 1 +
clang/lib/Sema/SemaOpenMP.cpp | 122 ++++++++
clang/lib/Sema/TreeTransform.h | 25 ++
clang/lib/Serialization/ASTReader.cpp | 8 +
clang/lib/Serialization/ASTWriter.cpp | 5 +
clang/test/OpenMP/replayable_ast_print.cpp | 233 ++++++++++++++
clang/test/OpenMP/replayable_messages.cpp | 51 ++++
.../taskgraph_replayable_false_codegen.cpp | 288 ++++++++++++++++++
.../taskgraph_replayable_restrictions.cpp | 173 +++++++++++
.../taskgraph_target_replayable_codegen.cpp | 126 ++++++++
clang/tools/libclang/CIndex.cpp | 4 +
llvm/include/llvm/Frontend/OpenMP/OMP.td | 18 ++
.../taskgraph_detach_dynamic_fails.cpp | 52 ++++
.../taskgraph/taskgraph_nested_replayable.cpp | 47 +++
.../taskgraph/taskgraph_replayable_false.cpp | 141 +++++++++
.../taskgraph_replayable_false_reduction.cpp | 88 ++++++
...layable_nonlexical_recursive_reduction.cpp | 147 +++++++++
..._nonlexical_reduction_minimal_taskloop.cpp | 105 +++++++
27 files changed, 1859 insertions(+), 49 deletions(-)
create mode 100644 clang/test/OpenMP/replayable_ast_print.cpp
create mode 100644 clang/test/OpenMP/replayable_messages.cpp
create mode 100644 clang/test/OpenMP/taskgraph_replayable_false_codegen.cpp
create mode 100644 clang/test/OpenMP/taskgraph_replayable_restrictions.cpp
create mode 100644 clang/test/OpenMP/taskgraph_target_replayable_codegen.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_detach_dynamic_fails.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_nested_replayable.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_false.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_false_reduction.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_recursive_reduction.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_reduction_minimal_taskloop.cpp
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index e046e639c1511d..8df512dffed3a2 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -2240,6 +2240,75 @@ class OMPSelfMapsClause final : public OMPClause {
}
};
+/// This represents a 'replayable' clause in the '#pragma omp target',
+// '#pragma omp target enter data', '#pragma omp target exit data',
+// '#pragma omp target update', '#pragma omp task', '#pragma omp taskloop' or
+// '#pragma omp taskwait' directive.
+///
+/// \code
+/// #pragma omp task replayable(1)
+/// \endcode
+/// In this example directive '#pragma omp task' has the 'replayable' clause.
+class OMPReplayableClause final : public OMPClause {
+ friend class OMPClauseReader;
+
+ /// Location of '('.
+ SourceLocation LParenLoc;
+
+ /// Condition of the 'replayable' clause.
+ Stmt *Condition = nullptr;
+
+public:
+ /// Build 'replayable' clause.
+ ///
+ /// \param Cond Condition of the clause.
+ /// \param StartLoc Starting location of the clause.
+ /// \param LParenLoc Location of '('.
+ /// \param EndLoc Ending location of the clause.
+ OMPReplayableClause(Expr *Cond, SourceLocation StartLoc,
+ SourceLocation LParenLoc, SourceLocation EndLoc)
+ : OMPClause(llvm::omp::OMPC_replayable, StartLoc, EndLoc),
+ LParenLoc(LParenLoc), Condition(Cond) {}
+
+ /// Build an empty clause.
+ OMPReplayableClause()
+ : OMPClause(llvm::omp::OMPC_replayable, SourceLocation(),
+ SourceLocation()) {}
+
+ /// Sets the location of '('.
+ void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+ /// Returns the location of '('.
+ SourceLocation getLParenLoc() const { return LParenLoc; }
+
+ /// Set condition.
+ void setCondition(Expr *Cond) { Condition = Cond; }
+
+ /// Returns condition.
+ Expr *getCondition() const { return cast_or_null<Expr>(Condition); }
+
+ child_range children() {
+ if (Condition)
+ return child_range(&Condition, &Condition + 1);
+ return child_range(child_iterator(), child_iterator());
+ }
+
+ const_child_range children() const {
+ if (Condition)
+ return const_child_range(&Condition, &Condition + 1);
+ return const_child_range(const_child_iterator(), const_child_iterator());
+ }
+
+ child_range used_children();
+ const_child_range used_children() const {
+ return const_cast<OMPReplayableClause *>(this)->used_children();
+ }
+
+ static bool classof(const OMPClause *T) {
+ return T->getClauseKind() == llvm::omp::OMPC_replayable;
+ }
+};
+
/// This represents 'at' clause in the '#pragma omp error' directive
///
/// \code
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 1913d095622f0e..c15b1fcc28816f 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3703,6 +3703,13 @@ bool RecursiveASTVisitor<Derived>::VisitOMPNowaitClause(OMPNowaitClause *C) {
return true;
}
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPReplayableClause(
+ OMPReplayableClause *C) {
+ TRY_TO(TraverseStmt(C->getCondition()));
+ return true;
+}
+
template <typename Derived>
bool RecursiveASTVisitor<Derived>::VisitOMPUntiedClause(OMPUntiedClause *) {
return true;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 82afe9922deb53..101c6482078935 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12492,6 +12492,8 @@ def err_omp_nowait_clause_without_depend: Error<
"directive '#pragma omp taskwait' cannot use 'nowait' clause without 'depend' clause">;
def err_omp_taskgraph_taskwait_without_depend: Error<
"directive '#pragma omp taskwait' within '#pragma omp taskgraph' must use 'depend' clause to be task-generating">;
+def err_omp_taskgraph_replayable_task: Error<
+ "%select{detachable|transparent|undeferred}0 replayable task is not allowed within '#pragma omp taskgraph'">;
def err_omp_taskgraph_not_task_generating: Error<
"'%0' directive is not task-generating and cannot be used within '#pragma omp taskgraph'">;
def note_omp_nowait_clause_here : Note<
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index ea601759b47217..e855b2a5eea6be 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1205,6 +1205,12 @@ class SemaOpenMP : public SemaBase {
OMPClause *ActOnOpenMPSelfMapsClause(SourceLocation StartLoc,
SourceLocation EndLoc);
+ /// Called on well-formed 'replayable' clause.
+ OMPClause *ActOnOpenMPReplayableClause(SourceLocation StartLoc,
+ SourceLocation EndLoc,
+ SourceLocation LParenLoc,
+ Expr *Condition);
+
/// Called on well-formed 'at' clause.
OMPClause *ActOnOpenMPAtClause(OpenMPAtClauseKind Kind,
SourceLocation KindLoc,
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 82e617f7b5cf0c..1d18473a3be405 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -335,6 +335,12 @@ OMPClause::child_range OMPNowaitClause::used_children() {
return children();
}
+OMPClause::child_range OMPReplayableClause::used_children() {
+ if (Condition)
+ return child_range(&Condition, &Condition + 1);
+ return children();
+}
+
OMPClause::child_range OMPGrainsizeClause::used_children() {
if (Stmt **C = getAddrOfExprAsWritten(getPreInitStmt()))
return child_range(C, C + 1);
@@ -2226,6 +2232,15 @@ void OMPClausePrinter::VisitOMPNowaitClause(OMPNowaitClause *Node) {
}
}
+void OMPClausePrinter::VisitOMPReplayableClause(OMPReplayableClause *Node) {
+ OS << "replayable";
+ if (auto *Cond = Node->getCondition()) {
+ OS << "(";
+ Cond->printPretty(OS, nullptr, Policy, 0);
+ OS << ")";
+ }
+}
+
void OMPClausePrinter::VisitOMPUntiedClause(OMPUntiedClause *) {
OS << "untied";
}
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index b898ac61226d68..603a1be8ec548a 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -628,6 +628,11 @@ void OMPClauseProfiler::VisitOMPNowaitClause(const OMPNowaitClause *C) {
Profiler->VisitStmt(C->getCondition());
}
+void OMPClauseProfiler::VisitOMPReplayableClause(const OMPReplayableClause *C) {
+ if (C->getCondition())
+ Profiler->VisitStmt(C->getCondition());
+}
+
void OMPClauseProfiler::VisitOMPUntiedClause(const OMPUntiedClause *) {}
void OMPClauseProfiler::VisitOMPMergeableClause(const OMPMergeableClause *) {}
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index a1dded426de409..492344606c7f56 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -4993,12 +4993,10 @@ void CGOpenMPRuntime::emitUpdateDependObjectsClause(
CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
}
-void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
- const OMPExecutableDirective &D,
- llvm::Function *TaskFunction,
- QualType SharedsTy, Address Shareds,
- const Expr *IfCond,
- const OMPTaskDataTy &Data) {
+void CGOpenMPRuntime::emitTaskCall(
+ CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
+ llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
+ const Expr *IfCond, const Expr *ReplayableCond, const OMPTaskDataTy &Data) {
if (!CGF.HaveInsertPoint())
return;
@@ -5160,22 +5158,21 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
}
};
- if (CGF.getOMPWithinTaskgraph()) {
- // Lexically within taskgraph, always replayable.
- RegionCodeGenTy TaskgraphRCG(TaskgraphTaskCodeGen);
- TaskgraphRCG(CGF);
+ // ReplayableCond is constant-folded to TRUE when lexically inside a taskgraph
+ // when a replayable clause is not present, else it's the clause's value.
+ if (ReplayableCond) {
+ emitIfClause(CGF, ReplayableCond, TaskgraphTaskCodeGen,
+ NonTaskgraphTaskCodeGen);
} else {
RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskCodeGen);
NonTaskgraphRCG(CGF);
}
}
-void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
- const OMPLoopDirective &D,
- llvm::Function *TaskFunction,
- QualType SharedsTy, Address Shareds,
- const Expr *IfCond,
- const OMPTaskDataTy &Data) {
+void CGOpenMPRuntime::emitTaskLoopCall(
+ CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
+ llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
+ const Expr *IfCond, const Expr *ReplayableCond, const OMPTaskDataTy &Data) {
if (!CGF.HaveInsertPoint())
return;
@@ -5353,10 +5350,11 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
TaskArgs);
};
- if (CGF.getOMPWithinTaskgraph()) {
- // Lexically within taskgraph, always replayable.
- RegionCodeGenTy TaskgraphRCG(TaskgraphTaskloopCodeGen);
- TaskgraphRCG(CGF);
+ // ReplayableCond is constant-folded to TRUE when lexically inside a taskgraph
+ // when a replayable clause is not present, else it's the clause's value.
+ if (ReplayableCond) {
+ emitIfClause(CGF, ReplayableCond, TaskgraphTaskloopCodeGen,
+ NonTaskgraphTaskloopCodeGen);
} else {
RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskloopCodeGen);
NonTaskgraphRCG(CGF);
@@ -6502,15 +6500,27 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
CGM.VoidPtrTy)};
- if (CGF.getOMPWithinTaskgraph())
+ // The __kmpc_taskgraph_taskred_init entry point allocates a copy of the
+ // reduction data, so only call it if there's a possibility that the construct
+ // may have 'replayable' true. In the case that the construct is false but we
+ // cannot tell until runtime, the allocated copy is freed safely in libomp.
+ bool MayBeReplayable;
+ if (Data.ReplayableCond) {
+ bool KnownValue;
+ MayBeReplayable =
+ !CGF.ConstantFoldsToSimpleInteger(Data.ReplayableCond, KnownValue) ||
+ KnownValue;
+ } else {
+ MayBeReplayable = CGF.getOMPWithinTaskgraph();
+ }
+ if (MayBeReplayable)
return CGF.EmitRuntimeCall(
OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), OMPRTL___kmpc_taskgraph_taskred_init),
Args);
- else
- return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_taskred_init),
- Args);
+ return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_taskred_init),
+ Args);
}
void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
@@ -6569,6 +6579,7 @@ Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
}
void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
+ const Expr *ReplayableCond,
const OMPTaskDataTy &Data) {
if (!CGF.HaveInsertPoint())
return;
@@ -6639,10 +6650,12 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
}
};
- if (CGF.getOMPWithinTaskgraph()) {
- // Lexically within taskgraph, always replayable.
- RegionCodeGenTy TaskgraphRCG(TaskgraphTaskwaitCodeGen);
- TaskgraphRCG(CGF);
+ // ReplayableCond is constant-folded to TRUE when lexically inside a
+ // taskgraph when a replayable clause is not present, else it's the clause's
+ // value.
+ if (ReplayableCond) {
+ emitIfClause(CGF, ReplayableCond, TaskgraphTaskwaitCodeGen,
+ NonTaskgraphTaskwaitCodeGen);
} else {
RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskwaitCodeGen);
NonTaskgraphRCG(CGF);
@@ -13920,19 +13933,17 @@ void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
llvm_unreachable("Not supported in SIMD-only mode");
}
-void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
- const OMPExecutableDirective &D,
- llvm::Function *TaskFunction,
- QualType SharedsTy, Address Shareds,
- const Expr *IfCond,
- const OMPTaskDataTy &Data) {
+void CGOpenMPSIMDRuntime::emitTaskCall(
+ CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
+ llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
+ const Expr *IfCond, const Expr *ReplayableCond, const OMPTaskDataTy &Data) {
llvm_unreachable("Not supported in SIMD-only mode");
}
void CGOpenMPSIMDRuntime::emitTaskLoopCall(
CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
- const Expr *IfCond, const OMPTaskDataTy &Data) {
+ const Expr *IfCond, const Expr *ReplayableCond, const OMPTaskDataTy &Data) {
llvm_unreachable("Not supported in SIMD-only mode");
}
@@ -13973,6 +13984,7 @@ Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
SourceLocation Loc,
+ const Expr *ReplayableCond,
const OMPTaskDataTy &Data) {
llvm_unreachable("Not supported in SIMD-only mode");
}
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 9dd6b3b02f7fdf..4d9a69fa1f1d9f 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -123,6 +123,7 @@ struct OMPTaskDataTy final {
bool IsWorksharingReduction = false;
bool HasNowaitClause = false;
bool HasModifier = false;
+ const Expr *ReplayableCond = nullptr;
};
/// Class intended to support codegen of all kind of the reduction clauses.
@@ -1177,6 +1178,7 @@ class CGOpenMPRuntime {
const OMPExecutableDirective &D,
llvm::Function *TaskFunction, QualType SharedsTy,
Address Shareds, const Expr *IfCond,
+ const Expr *ReplayableCond,
const OMPTaskDataTy &Data);
/// Emit task region for the taskloop directive. The taskloop region is
@@ -1212,7 +1214,8 @@ class CGOpenMPRuntime {
const OMPLoopDirective &D,
llvm::Function *TaskFunction,
QualType SharedsTy, Address Shareds,
- const Expr *IfCond, const OMPTaskDataTy &Data);
+ const Expr *IfCond, const Expr *ReplayableCond,
+ const OMPTaskDataTy &Data);
/// Emit code for the directive that does not require outlining.
///
@@ -1380,6 +1383,7 @@ class CGOpenMPRuntime {
/// Emit code for 'taskwait' directive.
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
+ const Expr *ReplayableCond,
const OMPTaskDataTy &Data);
/// Emit code for 'taskgraph' directive.
@@ -2066,6 +2070,7 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
const OMPExecutableDirective &D,
llvm::Function *TaskFunction, QualType SharedsTy,
Address Shareds, const Expr *IfCond,
+ const Expr *ReplayableCond,
const OMPTaskDataTy &Data) override;
/// Emit task region for the taskloop directive. The taskloop region is
@@ -2100,6 +2105,7 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
const OMPLoopDirective &D, llvm::Function *TaskFunction,
QualType SharedsTy, Address Shareds, const Expr *IfCond,
+ const Expr *ReplayableCond,
const OMPTaskDataTy &Data) override;
/// Emit a code for reduction clause. Next code should be emitted for
@@ -2220,6 +2226,7 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
/// Emit code for 'taskwait' directive.
void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
+ const Expr *ReplayableCond,
const OMPTaskDataTy &Data) override;
/// Emit code for 'taskgraph' directive.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 91a85b4c26c338..a062c86f080d89 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -6024,8 +6024,19 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
IntegerLiteral IfCond(getContext(), TrueOrFalse,
getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
SourceLocation());
+ const Expr *ReplayableCond = nullptr;
+ if (auto *RC = S.getSingleClause<OMPReplayableClause>()) {
+ ReplayableCond = RC->getCondition();
+ if (!ReplayableCond) {
+ ReplayableCond = IntegerLiteral::Create(
+ getContext(), llvm::APInt(32, 1),
+ getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
+ SourceLocation());
+ }
+ }
CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
- SharedsTy, CapturedStruct, &IfCond, Data);
+ SharedsTy, CapturedStruct, &IfCond,
+ ReplayableCond, Data);
}
void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
@@ -6116,6 +6127,35 @@ void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
(void)InRedScope.Privatize();
}
+/// Return the condition that decides whether \p S is a replayable construct,
+/// or null if it cannot be one.
+///
+/// OpenMP 6.0 [14.3] makes a task-generating construct that is encountered in
+/// a taskgraph construct a replayable construct of the region "unless
+/// otherwise specified by the replayable clause". So the enclosing taskgraph
+/// supplies the default and an explicit clause overrides it, rather than the
+/// taskgraph forcing every construct within it to be recorded. A clause whose
+/// argument is omitted means replayable, as does the taskgraph default; both
+/// are returned as a constant-true condition, which emitIfClause folds away.
+///
+/// [14.6] gives replayable-expression the constant property, so in a
+/// conforming program the result here is always a constant and no branch
+/// survives. A non-constant argument is accepted as an extension and decided
+/// at run time.
+static const Expr *getOMPReplayableCond(CodeGenFunction &CGF,
+ const OMPExecutableDirective &S) {
+ const auto *RC = S.getSingleClause<OMPReplayableClause>();
+ if (!RC && !CGF.getOMPWithinTaskgraph())
+ return nullptr;
+ if (RC)
+ if (const Expr *Cond = RC->getCondition())
+ return Cond;
+ return IntegerLiteral::Create(
+ CGF.getContext(), llvm::APInt(32, 1),
+ CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
+ SourceLocation());
+}
+
void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
// Emit outlined function for task construct.
const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
@@ -6134,15 +6174,20 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
OMPTaskDataTy Data;
// Check if we should emit tied or untied task.
Data.Tied = !S.getSingleClause<OMPUntiedClause>();
+ const Expr *ReplayableCond = getOMPReplayableCond(*this, S);
+ // Propagate the replayable signal into Data so that reduction init can
+ // route to the taskgraph-aware runtime entry point when the task may
+ // participate in taskgraph replay.
+ Data.ReplayableCond = ReplayableCond;
auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
CGF.EmitStmt(CS->getCapturedStmt());
};
- auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
- IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
- const OMPTaskDataTy &Data) {
+ auto &&TaskGen = [&S, SharedsTy, CapturedStruct, IfCond, ReplayableCond](
+ CodeGenFunction &CGF, llvm::Function *OutlinedFn,
+ const OMPTaskDataTy &Data) {
CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
SharedsTy, CapturedStruct, IfCond,
- Data);
+ ReplayableCond, Data);
};
auto LPCRegion =
CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
@@ -6173,7 +6218,8 @@ void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
// Build list of dependences
buildDependences(S, Data);
Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
- CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);
+ CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(),
+ getOMPReplayableCond(*this, S), Data);
}
void CodeGenFunction::EmitOMPTaskgraphDirective(
@@ -8577,11 +8623,17 @@ void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
}
}
+ const Expr *ReplayableCond = getOMPReplayableCond(*this, S);
+
OMPTaskDataTy Data;
// Check if taskloop must be emitted without taskgroup.
Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
// TODO: Check if we should emit tied or untied task.
Data.Tied = true;
+ // Propagate the replayable signal into Data so that reduction init can
+ // route to the taskgraph-aware runtime entry point when the taskloop may
+ // participate in taskgraph replay.
+ Data.ReplayableCond = ReplayableCond;
// Set scheduling for taskloop
if (const auto *Clause = S.getSingleClause<OMPGrainsizeClause>()) {
// grainsize clause
@@ -8696,15 +8748,16 @@ void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
(*LIP)->getType(), S.getBeginLoc()));
});
};
- auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
- IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
- const OMPTaskDataTy &Data) {
+ auto &&TaskGen = [&S, SharedsTy, CapturedStruct, IfCond, ReplayableCond](
+ CodeGenFunction &CGF, llvm::Function *OutlinedFn,
+ const OMPTaskDataTy &Data) {
auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
+ ReplayableCond,
&Data](CodeGenFunction &CGF, PrePostActionTy &) {
OMPLoopScope PreInitScope(CGF, S);
- CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
- OutlinedFn, SharedsTy,
- CapturedStruct, IfCond, Data);
+ CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(
+ CGF, S.getBeginLoc(), S, OutlinedFn, SharedsTy, CapturedStruct,
+ IfCond, ReplayableCond, Data);
};
CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
CodeGen);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 1741af8c7e6053..893bb06b749277 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3532,6 +3532,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
Clause = ParseOpenMPClause(CKind, WrongDirective);
break;
case OMPC_graph_reset:
+ case OMPC_replayable:
if (!FirstClause) {
Diag(Tok, diag::err_omp_more_one_clause)
<< getOpenMPDirectiveName(DKind, OMPVersion)
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 58078a6a43a577..7214e2e7c50da2 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -5818,6 +5818,95 @@ static bool checkIfClauses(Sema &S, OpenMPDirectiveKind Kind,
return ErrorFound;
}
+// If we're inside a taskgraph region, determine if a construct is (or might be)
+// replayable.
+static bool isReplayableConstruct(Sema &S, ArrayRef<OMPClause *> Clauses) {
+ const auto *RC =
+ OMPExecutableDirective::getSingleClause<OMPReplayableClause>(Clauses);
+ if (!RC)
+ return true;
+ // An omitted argument means replayable.
+ const Expr *Cond = RC->getCondition();
+ if (!Cond)
+ return true;
+ bool Result;
+ if (!Cond->EvaluateAsBooleanCondition(Result, S.getASTContext()))
+ return true;
+ return Result;
+}
+
+// OpenMP 6.0 [14.3, taskgraph Construct, Restrictions] Some tasks cannot be
+// recorded as a graph node and replayed: this applies to detachable,
+// transparent and undeferred tasks. This restriction applies only to a
+// replayable construct, so can be overridden with replayable(false). Kept in
+// step with the flang checks in OmpStructureChecker's TaskgraphVisitor
+// (check-omp-structure.cpp).
+static bool
+checkTaskgraphReplayableRestrictions(Sema &S, OpenMPDirectiveKind Kind,
+ ArrayRef<OMPClause *> Clauses) {
+ // A dependent replayable condition is left to the instantiation, which comes
+ // back through here with the argument substituted. Deciding now would
+ // diagnose every instantiation, including the ones that opt out.
+ if (const auto *RC =
+ OMPExecutableDirective::getSingleClause<OMPReplayableClause>(Clauses))
+ if (const Expr *Cond = RC->getCondition())
+ if (Cond->isValueDependent())
+ return false;
+
+ if (!isReplayableConstruct(S, Clauses))
+ return false;
+
+ // The if clause only generates an undeferred *task* on these two, and the
+ // body of the generated task is not part of the taskgraph region.
+ OpenMPDirectiveKind Leaf = getLeafConstructsOrSelf(Kind).front();
+ bool IsTaskGenerating = Leaf == OMPD_task || Leaf == OMPD_taskloop;
+
+ enum { Detachable, Transparent, Undeferred };
+ bool ErrorFound = false;
+ for (const OMPClause *C : Clauses) {
+ std::optional<unsigned> Which;
+ switch (C->getClauseKind()) {
+ case OMPC_detach:
+ Which = Detachable;
+ break;
+ case OMPC_transparent: {
+ const Expr *Impex = cast<OMPTransparentClause>(C)->getImpexType();
+ if (!Impex) {
+ Which = Transparent;
+ break;
+ }
+ Expr::EvalResult Eval;
+ if (!Impex->isValueDependent() &&
+ Impex->EvaluateAsInt(Eval, S.getASTContext()) &&
+ !Eval.Val.getInt().isZero())
+ Which = Transparent;
+ break;
+ }
+ case OMPC_if: {
+ const auto *IC = cast<OMPIfClause>(C);
+ OpenMPDirectiveKind NM = IC->getNameModifier();
+ if (!IsTaskGenerating || (NM != OMPD_unknown && NM != Leaf))
+ break;
+ const Expr *Cond = IC->getCondition();
+ bool Result;
+ if (Cond && !Cond->isValueDependent() &&
+ Cond->EvaluateAsBooleanCondition(Result, S.getASTContext()) &&
+ !Result)
+ Which = Undeferred;
+ break;
+ }
+ default:
+ break;
+ }
+ if (Which) {
+ S.Diag(C->getBeginLoc(), diag::err_omp_taskgraph_replayable_task)
+ << *Which << SourceRange(C->getBeginLoc(), C->getEndLoc());
+ ErrorFound = true;
+ }
+ }
+ return ErrorFound;
+}
+
static std::pair<ValueDecl *, bool>
getPrivateItem(Sema &S, Expr *&RefExpr, SourceLocation &ELoc,
SourceRange &ERange, bool AllowArraySection,
@@ -7285,6 +7374,7 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
case OMPC_graph_id:
case OMPC_graph_reset:
case OMPC_nocontext:
+ case OMPC_replayable:
// Do not analyze if no parent parallel directive.
if (isOpenMPParallelDirective(Kind))
break;
@@ -7407,6 +7497,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
ErrorFound = checkIfClauses(SemaRef, Kind, Clauses, AllowedNameModifiers) ||
ErrorFound;
+ if (DSAStack->getParentDirective() == OMPD_taskgraph)
+ ErrorFound = checkTaskgraphReplayableRestrictions(SemaRef, Kind, Clauses) ||
+ ErrorFound;
+
if (ErrorFound)
return StmtError();
@@ -18444,6 +18538,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
case OMPC_graph_reset:
Res = ActOnOpenMPGraphResetClause(Expr, StartLoc, LParenLoc, EndLoc);
break;
+ case OMPC_replayable:
+ Res = ActOnOpenMPReplayableClause(StartLoc, EndLoc, LParenLoc, Expr);
+ break;
case OMPC_novariants:
Res = ActOnOpenMPNovariantsClause(Expr, StartLoc, LParenLoc, EndLoc);
break;
@@ -20250,6 +20347,11 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
case OMPC_self_maps:
Res = ActOnOpenMPSelfMapsClause(StartLoc, EndLoc);
break;
+ case OMPC_replayable:
+ Res = ActOnOpenMPReplayableClause(StartLoc, EndLoc,
+ /*LParenLoc=*/SourceLocation(),
+ /*Condition=*/nullptr);
+ break;
case OMPC_destroy:
Res = ActOnOpenMPDestroyClause(/*InteropVar=*/nullptr, StartLoc,
/*LParenLoc=*/SourceLocation(),
@@ -20488,6 +20590,26 @@ OMPClause *SemaOpenMP::ActOnOpenMPSelfMapsClause(SourceLocation StartLoc,
return new (getASTContext()) OMPSelfMapsClause(StartLoc, EndLoc);
}
+OMPClause *SemaOpenMP::ActOnOpenMPReplayableClause(SourceLocation StartLoc,
+ SourceLocation EndLoc,
+ SourceLocation LParenLoc,
+ Expr *Condition) {
+ Expr *ValExpr = Condition;
+ if (Condition && LParenLoc.isValid()) {
+ if (!Condition->isValueDependent() && !Condition->isTypeDependent() &&
+ !Condition->isInstantiationDependent() &&
+ !Condition->containsUnexpandedParameterPack()) {
+ ExprResult Val = SemaRef.CheckBooleanCondition(StartLoc, Condition);
+ if (Val.isInvalid())
+ return nullptr;
+
+ ValExpr = Val.get();
+ }
+ }
+ return new (getASTContext())
+ OMPReplayableClause(ValExpr, StartLoc, LParenLoc, EndLoc);
+}
+
StmtResult
SemaOpenMP::ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
SourceLocation StartLoc,
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 27070b434c8655..da158f2791f0c3 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1931,6 +1931,18 @@ class TreeTransform {
LParenLoc, Condition);
}
+ /// Build a new OpenMP 'replayable' clause.
+ ///
+ /// By default, performs semantic analysis to build the new OpenMP clause.
+ /// Subclasses may override this routine to provide different behavior.
+ OMPClause *RebuildOMPReplayableClause(Expr *Condition,
+ SourceLocation StartLoc,
+ SourceLocation LParenLoc,
+ SourceLocation EndLoc) {
+ return getSema().OpenMP().ActOnOpenMPReplayableClause(StartLoc, EndLoc,
+ LParenLoc, Condition);
+ }
+
/// Build a new OpenMP 'private' clause.
///
/// By default, performs semantic analysis to build the new OpenMP clause.
@@ -11303,6 +11315,19 @@ TreeTransform<Derived>::TransformOMPNowaitClause(OMPNowaitClause *C) {
C->getLParenLoc(), C->getEndLoc());
}
+template <typename Derived>
+OMPClause *
+TreeTransform<Derived>::TransformOMPReplayableClause(OMPReplayableClause *C) {
+ ExprResult Cond;
+ if (auto *Condition = C->getCondition()) {
+ Cond = getDerived().TransformExpr(Condition);
+ if (Cond.isInvalid())
+ return nullptr;
+ }
+ return getDerived().RebuildOMPReplayableClause(
+ Cond.get(), C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
+}
+
template <typename Derived>
OMPClause *
TreeTransform<Derived>::TransformOMPUntiedClause(OMPUntiedClause *C) {
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 7f2a811eaf7f24..7588b3fa61aef9 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11906,6 +11906,9 @@ OMPClause *OMPClauseReader::readClause() {
case llvm::omp::OMPC_graph_reset:
C = new (Context) OMPGraphResetClause();
break;
+ case llvm::omp::OMPC_replayable:
+ C = new (Context) OMPReplayableClause();
+ break;
case llvm::omp::OMPC_num_tasks:
C = new (Context) OMPNumTasksClause();
break;
@@ -12228,6 +12231,11 @@ void OMPClauseReader::VisitOMPNowaitClause(OMPNowaitClause *C) {
C->setLParenLoc(Record.readSourceLocation());
}
+void OMPClauseReader::VisitOMPReplayableClause(OMPReplayableClause *C) {
+ C->setCondition(Record.readSubExpr());
+ C->setLParenLoc(Record.readSourceLocation());
+}
+
void OMPClauseReader::VisitOMPUntiedClause(OMPUntiedClause *) {}
void OMPClauseReader::VisitOMPMergeableClause(OMPMergeableClause *) {}
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index c0b77441e4f290..cfcfbaa763b3b0 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8302,6 +8302,11 @@ void OMPClauseWriter::VisitOMPNowaitClause(OMPNowaitClause *C) {
Record.AddSourceLocation(C->getLParenLoc());
}
+void OMPClauseWriter::VisitOMPReplayableClause(OMPReplayableClause *C) {
+ Record.AddStmt(C->getCondition());
+ Record.AddSourceLocation(C->getLParenLoc());
+}
+
void OMPClauseWriter::VisitOMPUntiedClause(OMPUntiedClause *) {}
void OMPClauseWriter::VisitOMPMergeableClause(OMPMergeableClause *) {}
diff --git a/clang/test/OpenMP/replayable_ast_print.cpp b/clang/test/OpenMP/replayable_ast_print.cpp
new file mode 100644
index 00000000000000..cea97b8b1cd736
--- /dev/null
+++ b/clang/test/OpenMP/replayable_ast_print.cpp
@@ -0,0 +1,233 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-dump %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+void replayable_clauses() {
+ int A = 1;
+ int B[10];
+
+ // --- omp task ---
+
+ // DUMP: OMPTaskDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // PRINT: #pragma omp task replayable
+ #pragma omp task replayable
+ {}
+
+ // DUMP: OMPTaskDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+ // PRINT: #pragma omp task replayable(false)
+ #pragma omp task replayable(false)
+ {}
+
+ // DUMP: OMPTaskDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+ // PRINT: #pragma omp task replayable(true)
+ #pragma omp task replayable(true)
+ {}
+
+ // DUMP: OMPTaskDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+ // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+ // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+ // PRINT: #pragma omp task replayable(A > 5)
+ #pragma omp task replayable(A > 5)
+ {}
+
+ // --- omp taskloop ---
+
+ // DUMP: OMPTaskLoopDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // PRINT: #pragma omp taskloop replayable
+ #pragma omp taskloop replayable
+ for (int i = 0; i < 10; ++i)
+ {}
+
+ // DUMP: OMPTaskLoopDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+ // PRINT: #pragma omp taskloop replayable(false)
+ #pragma omp taskloop replayable(false)
+ for (int i = 0; i < 10; ++i)
+ {}
+
+ // DUMP: OMPTaskLoopDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+ // PRINT: #pragma omp taskloop replayable(true)
+ #pragma omp taskloop replayable(true)
+ for (int i = 0; i < 10; ++i)
+ {}
+
+ // DUMP: OMPTaskLoopDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+ // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+ // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+ // PRINT: #pragma omp taskloop replayable(A > 5)
+ #pragma omp taskloop replayable(A > 5)
+ for (int i = 0; i < 10; ++i)
+ {}
+
+ // --- omp taskwait ---
+
+ // DUMP: OMPTaskwaitDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // PRINT: #pragma omp taskwait replayable
+ #pragma omp taskwait replayable
+
+ // DUMP: OMPTaskwaitDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+ // PRINT: #pragma omp taskwait replayable(false)
+ #pragma omp taskwait replayable(false)
+
+ // DUMP: OMPTaskwaitDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+ // PRINT: #pragma omp taskwait replayable(true)
+ #pragma omp taskwait replayable(true)
+
+ // DUMP: OMPTaskwaitDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+ // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+ // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+ // PRINT: #pragma omp taskwait replayable(A > 5)
+ #pragma omp taskwait replayable(A > 5)
+
+ // --- omp target ---
+
+ // DUMP: OMPTargetDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // PRINT: #pragma omp target replayable
+ #pragma omp target replayable
+ {}
+
+ // DUMP: OMPTargetDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+ // PRINT: #pragma omp target replayable(false)
+ #pragma omp target replayable(false)
+ {}
+
+ // DUMP: OMPTargetDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+ // PRINT: #pragma omp target replayable(true)
+ #pragma omp target replayable(true)
+ {}
+
+ // DUMP: OMPTargetDirective
+ // DUMP-NEXT: OMPReplayableClause
+ // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+ // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+ // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+ // PRINT: #pragma omp target replayable(A > 5)
+ #pragma omp target replayable(A > 5)
+ {}
+
+ // --- omp target enter data ---
+
+ // DUMP: OMPTargetEnterDataDirective
+ // DUMP: OMPReplayableClause
+ // PRINT: #pragma omp target enter data map(to: A) replayable
+ #pragma omp target enter data map(to: A) replayable
+
+ // DUMP: OMPTargetEnterDataDirective
+ // DUMP: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+ // PRINT: #pragma omp target enter data map(to: A) replayable(false)
+ #pragma omp target enter data map(to: A) replayable(false)
+
+ // DUMP: OMPTargetEnterDataDirective
+ // DUMP: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+ // PRINT: #pragma omp target enter data map(to: A) replayable(true)
+ #pragma omp target enter data map(to: A) replayable(true)
+
+ // DUMP: OMPTargetEnterDataDirective
+ // DUMP: OMPReplayableClause
+ // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+ // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+ // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+ // PRINT: #pragma omp target enter data map(to: A) replayable(A > 5)
+ #pragma omp target enter data map(to: A) replayable(A > 5)
+
+ // --- omp target exit data ---
+
+ // DUMP: OMPTargetExitDataDirective
+ // DUMP: OMPReplayableClause
+ // PRINT: #pragma omp target exit data map(from: A) replayable
+ #pragma omp target exit data map(from: A) replayable
+
+ // DUMP: OMPTargetExitDataDirective
+ // DUMP: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+ // PRINT: #pragma omp target exit data map(from: A) replayable(false)
+ #pragma omp target exit data map(from: A) replayable(false)
+
+ // DUMP: OMPTargetExitDataDirective
+ // DUMP: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+ // PRINT: #pragma omp target exit data map(from: A) replayable(true)
+ #pragma omp target exit data map(from: A) replayable(true)
+
+ // DUMP: OMPTargetExitDataDirective
+ // DUMP: OMPReplayableClause
+ // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+ // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+ // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+ // PRINT: #pragma omp target exit data map(from: A) replayable(A > 5)
+ #pragma omp target exit data map(from: A) replayable(A > 5)
+
+ // --- omp target update ---
+
+ // DUMP: OMPTargetUpdateDirective
+ // DUMP: OMPReplayableClause
+ // PRINT: #pragma omp target update to(A) replayable
+ #pragma omp target update to(A) replayable
+
+ // DUMP: OMPTargetUpdateDirective
+ // DUMP: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+ // PRINT: #pragma omp target update to(A) replayable(false)
+ #pragma omp target update to(A) replayable(false)
+
+ // DUMP: OMPTargetUpdateDirective
+ // DUMP: OMPReplayableClause
+ // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+ // PRINT: #pragma omp target update to(A) replayable(true)
+ #pragma omp target update to(A) replayable(true)
+
+ // DUMP: OMPTargetUpdateDirective
+ // DUMP: OMPReplayableClause
+ // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+ // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+ // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+ // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+ // PRINT: #pragma omp target update to(A) replayable(A > 5)
+ #pragma omp target update to(A) replayable(A > 5)
+}
+#endif
diff --git a/clang/test/OpenMP/replayable_messages.cpp b/clang/test/OpenMP/replayable_messages.cpp
new file mode 100644
index 00000000000000..10172f50162ade
--- /dev/null
+++ b/clang/test/OpenMP/replayable_messages.cpp
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=51 -verify=expected,omp51 -fsyntax-only %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -verify=expected,omp60 -fsyntax-only %s
+
+// Tests that the 'replayable' clause is accepted in OpenMP 6.0 and rejected in
+// prior versions on all seven supported directives. Also tests duplicate-clause
+// and invalid-condition diagnostics.
+
+void foo() {}
+
+void replayable_messages() {
+ int A = 1;
+
+ #pragma omp task replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp task'}}
+ {}
+
+ #pragma omp taskloop replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp taskloop'}}
+ for (int i = 0; i < 10; ++i)
+ {}
+
+ #pragma omp taskwait replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp taskwait'}}
+
+ #pragma omp target replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target'}}
+ {}
+
+ #pragma omp target enter data map(to: A) replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target enter data'}}
+
+ #pragma omp target exit data map(from: A) replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target exit data'}}
+
+ #pragma omp target update to(A) replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target update'}}
+
+ #pragma omp task replayable replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp task'}} omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp task'}} expected-error {{directive '#pragma omp task' cannot contain more than one 'replayable' clause}}
+ {}
+
+ #pragma omp task replayable(foo()) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp task'}} omp60-error {{value of type 'void' is not contextually convertible to 'bool'}}
+ {}
+
+ #pragma omp taskloop replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp taskloop'}}
+ for (int i = 0; i < 10; ++i)
+ {}
+
+ #pragma omp taskwait replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp taskwait'}}
+
+ #pragma omp target replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target'}}
+ {}
+
+ #pragma omp target enter data map(to: A) replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target enter data'}}
+
+ #pragma omp target exit data map(from: A) replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target exit data'}}
+
+ #pragma omp target update to(A) replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target update'}}
+}
diff --git a/clang/test/OpenMP/taskgraph_replayable_false_codegen.cpp b/clang/test/OpenMP/taskgraph_replayable_false_codegen.cpp
new file mode 100644
index 00000000000000..0349943a4d15d2
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_replayable_false_codegen.cpp
@@ -0,0 +1,288 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-linux-gnu -fopenmp-targets=x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+// OpenMP 6.0 [14.3]: a task-generating construct encountered in a taskgraph
+// construct is a replayable construct of the region "unless otherwise
+// specified by the replayable clause". So the enclosing taskgraph supplies the
+// default and the clause overrides it: replayable(false) inside a taskgraph
+// region takes the construct out of the record, leaving it to be encountered
+// normally on the (recording) execution of the region and, since a replay
+// execution does not execute the region body, omitted from every replay.
+//
+// Checked for each directive that admits the clause: the default records via a
+// __kmpc_taskgraph* entry point, replayable(false) takes the ordinary path
+// instead, and a non-constant argument -- a clang extension, since [14.6]
+// gives the argument the constant property -- selects between the two at run
+// time.
+
+#ifndef HEADER
+#define HEADER
+
+void body();
+
+// --- task -------------------------------------------------------------------
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}task_defaultv
+void task_default() {
+ // CHECK: call i32 @__kmpc_taskgraph_task(
+ // CHECK-NOT: call i32 @__kmpc_omp_task(
+#pragma omp taskgraph
+ {
+#pragma omp task
+ body();
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}task_replayable_onev
+void task_replayable_one() {
+ // An explicit true argument agrees with the default the taskgraph supplies.
+ // CHECK: call i32 @__kmpc_taskgraph_task(
+ // CHECK-NOT: call i32 @__kmpc_omp_task(
+#pragma omp taskgraph
+ {
+#pragma omp task replayable(1)
+ body();
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}task_replayable_falsev
+void task_replayable_false() {
+ // CHECK: call i32 @__kmpc_omp_task(
+ // CHECK-NOT: call i32 @__kmpc_taskgraph_task(
+#pragma omp taskgraph
+ {
+#pragma omp task replayable(false)
+ body();
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}task_replayable_dyni
+void task_replayable_dyn(int c) {
+ // CHECK: br i1
+ // CHECK-DAG: call i32 @__kmpc_taskgraph_task(
+ // CHECK-DAG: call i32 @__kmpc_omp_task(
+#pragma omp taskgraph
+ {
+#pragma omp task replayable(c)
+ body();
+ }
+}
+
+// Outside any taskgraph region there is no default to override: a task with no
+// clause, and one that opts out, both take the ordinary path.
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}task_outside_defaultv
+void task_outside_default() {
+ // CHECK: call i32 @__kmpc_omp_task(
+ // CHECK-NOT: call i32 @__kmpc_taskgraph_task(
+#pragma omp task
+ body();
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}task_outside_replayable_falsev
+void task_outside_replayable_false() {
+ // CHECK: call i32 @__kmpc_omp_task(
+ // CHECK-NOT: call i32 @__kmpc_taskgraph_task(
+#pragma omp task replayable(false)
+ body();
+}
+
+// --- taskloop ---------------------------------------------------------------
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}taskloop_defaultv
+void taskloop_default() {
+ // CHECK: call i32 @__kmpc_taskgraph_taskloop(
+ // CHECK-NOT: call void @__kmpc_taskloop(
+#pragma omp taskgraph
+ {
+#pragma omp taskloop
+ for (int i = 0; i < 4; ++i)
+ body();
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}taskloop_replayable_falsev
+void taskloop_replayable_false() {
+ // CHECK: call void @__kmpc_taskloop(
+ // CHECK-NOT: call i32 @__kmpc_taskgraph_taskloop(
+#pragma omp taskgraph
+ {
+#pragma omp taskloop replayable(false)
+ for (int i = 0; i < 4; ++i)
+ body();
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}taskloop_replayable_dyni
+void taskloop_replayable_dyn(int c) {
+ // CHECK: br i1
+ // CHECK-DAG: call i32 @__kmpc_taskgraph_taskloop(
+ // CHECK-DAG: call void @__kmpc_taskloop(
+#pragma omp taskgraph
+ {
+#pragma omp taskloop replayable(c)
+ for (int i = 0; i < 4; ++i)
+ body();
+ }
+}
+
+// --- taskwait ---------------------------------------------------------------
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}taskwait_defaultv
+void taskwait_default() {
+ int x = 0;
+ // CHECK: call void @__kmpc_taskgraph_taskwait(
+ // CHECK-NOT: call void @__kmpc_omp_taskwait_deps_51(
+#pragma omp taskgraph
+ {
+#pragma omp taskwait depend(inout : x)
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}taskwait_replayable_falsev
+void taskwait_replayable_false() {
+ int x = 0;
+ // CHECK: call void @__kmpc_omp_taskwait_deps_51(
+ // CHECK-NOT: call void @__kmpc_taskgraph_taskwait(
+#pragma omp taskgraph
+ {
+#pragma omp taskwait depend(inout : x) replayable(false)
+ }
+}
+
+// --- target -----------------------------------------------------------------
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}target_defaultv
+void target_default() {
+ int x = 0;
+ // CHECK: call i32 @__kmpc_taskgraph_target(
+ // CHECK-NOT: call i32 @__tgt_target_kernel(
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : x)
+ { x++; }
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}target_replayable_falsev
+void target_replayable_false() {
+ int x = 0;
+ // CHECK: call i32 @__tgt_target_kernel(
+ // CHECK-NOT: call i32 @__kmpc_taskgraph_target(
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : x) replayable(false)
+ { x++; }
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}target_replayable_dyni
+void target_replayable_dyn(int c) {
+ int x = 0;
+ // CHECK: br i1
+ // CHECK-DAG: call i32 @__kmpc_taskgraph_target(
+ // CHECK-DAG: call i32 @__tgt_target_kernel(
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : x) replayable(c)
+ { x++; }
+ }
+}
+
+// A recorded target forwards its dependences to __kmpc_taskgraph_target, which
+// satisfies them synchronously, so it needs no hidden helper task. One that
+// opts out needs the helper task back: it takes the ordinary launch path, where
+// the depend clause has nothing else to satisfy it.
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}target_depend_defaultv
+void target_depend_default() {
+ int x = 0, d = 0;
+ // CHECK-NOT: @__kmpc_omp_task_alloc(
+ // CHECK: call i32 @__kmpc_taskgraph_target(ptr {{[^,]*}}, i32 {{[^,]*}}, i32 1, ptr
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : x) depend(inout : d)
+ { x++; }
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}target_depend_replayable_falsev
+void target_depend_replayable_false() {
+ int x = 0, d = 0;
+ // CHECK: call ptr @__kmpc_omp_task_alloc(
+ // CHECK: call void @__kmpc_omp_taskwait_deps_51(
+ // CHECK-NOT: call i32 @__kmpc_taskgraph_target(
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : x) depend(inout : d) replayable(false)
+ { x++; }
+ }
+}
+
+// --- target enter/exit data, target update ----------------------------------
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}enter_data_defaultv
+void enter_data_default() {
+ int x = 0;
+ // CHECK: call void @__kmpc_taskgraph_target_enter_data(
+ // CHECK-NOT: call void @__tgt_target_data_begin_mapper(
+#pragma omp taskgraph
+ {
+#pragma omp target enter data map(to : x)
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}enter_data_replayable_falsev
+void enter_data_replayable_false() {
+ int x = 0;
+ // CHECK: call void @__tgt_target_data_begin_mapper(
+ // CHECK-NOT: call void @__kmpc_taskgraph_target_enter_data(
+#pragma omp taskgraph
+ {
+#pragma omp target enter data map(to : x) replayable(false)
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}exit_data_defaultv
+void exit_data_default() {
+ int x = 0;
+ // CHECK: call void @__kmpc_taskgraph_target_exit_data(
+ // CHECK-NOT: call void @__tgt_target_data_end_mapper(
+#pragma omp taskgraph
+ {
+#pragma omp target exit data map(from : x)
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}exit_data_replayable_falsev
+void exit_data_replayable_false() {
+ int x = 0;
+ // CHECK: call void @__tgt_target_data_end_mapper(
+ // CHECK-NOT: call void @__kmpc_taskgraph_target_exit_data(
+#pragma omp taskgraph
+ {
+#pragma omp target exit data map(from : x) replayable(false)
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}update_defaultv
+void update_default() {
+ int x = 0;
+ // CHECK: call void @__kmpc_taskgraph_target_update(
+ // CHECK-NOT: call void @__tgt_target_data_update_mapper(
+#pragma omp taskgraph
+ {
+#pragma omp target update to(x)
+ }
+}
+
+// CHECK-LABEL: define {{.*}} @_Z{{[0-9]+}}update_replayable_falsev
+void update_replayable_false() {
+ int x = 0;
+ // CHECK: call void @__tgt_target_data_update_mapper(
+ // CHECK-NOT: call void @__kmpc_taskgraph_target_update(
+#pragma omp taskgraph
+ {
+#pragma omp target update to(x) replayable(false)
+ }
+}
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_replayable_restrictions.cpp b/clang/test/OpenMP/taskgraph_replayable_restrictions.cpp
new file mode 100644
index 00000000000000..0d59fbebf37bd1
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_replayable_restrictions.cpp
@@ -0,0 +1,173 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -verify -fsyntax-only %s
+
+// OpenMP 6.0 [14.3]: a replayable construct in a taskgraph region must not
+// generate a detachable, transparent or undeferred task. Each restriction is
+// on the replayable construct, so replayable(false) is the way to write one of
+// these inside a taskgraph region. The same cases are covered for flang in
+// flang/test/Semantics/OpenMP/taskgraph.f90.
+
+typedef unsigned long omp_event_handle_t;
+
+// The impex constants are not required to be constant expressions in C/C++,
+// so a 'transparent' argument the compiler cannot read is left alone. Spell
+// them as an enum here to exercise the cases where it can.
+enum omp_impex_t {
+ omp_not_impex = 0,
+ omp_import = 1,
+ omp_export = 2,
+ omp_impex = 3
+};
+
+void detachable() {
+ omp_event_handle_t ev;
+
+#pragma omp taskgraph
+ {
+#pragma omp task detach(ev) // expected-error {{detachable replayable task is not allowed within '#pragma omp taskgraph'}}
+ {}
+ }
+
+ // Ok: detachable, but not replayable.
+#pragma omp taskgraph
+ {
+#pragma omp task detach(ev) replayable(0)
+ {}
+ }
+
+ // Ok: outside any taskgraph region.
+#pragma omp task detach(ev)
+ {}
+}
+
+void transparent(enum omp_impex_t which) {
+#pragma omp taskgraph
+ {
+#pragma omp task transparent // expected-error {{transparent replayable task is not allowed within '#pragma omp taskgraph'}}
+ {}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp task transparent(omp_impex) // expected-error {{transparent replayable task is not allowed within '#pragma omp taskgraph'}}
+ {}
+ }
+
+ // Ok: omp_not_impex leaves the task non-transparent.
+#pragma omp taskgraph
+ {
+#pragma omp task transparent(omp_not_impex)
+ {}
+ }
+
+ // Ok: transparent, but not replayable.
+#pragma omp taskgraph
+ {
+#pragma omp task transparent replayable(0)
+ {}
+ }
+
+ // Ok: the impex value is not known here, so it is not diagnosed.
+#pragma omp taskgraph
+ {
+#pragma omp task transparent(which)
+ {}
+ }
+}
+
+void undeferred(int cond) {
+#pragma omp taskgraph
+ {
+#pragma omp task if (0) // expected-error {{undeferred replayable task is not allowed within '#pragma omp taskgraph'}}
+ {}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp task if (task : 0) // expected-error {{undeferred replayable task is not allowed within '#pragma omp taskgraph'}}
+ {}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp taskloop if (0) // expected-error {{undeferred replayable task is not allowed within '#pragma omp taskgraph'}}
+ for (int i = 0; i < 4; ++i)
+ ;
+ }
+
+ // Ok: the condition is not known to be false, so no undeferred task is
+ // necessarily generated. A runtime that records one anyway is on its own.
+#pragma omp taskgraph
+ {
+#pragma omp task if (cond)
+ {}
+ }
+
+ // Ok: true is a deferred task.
+#pragma omp taskgraph
+ {
+#pragma omp task if (1)
+ {}
+ }
+
+ // Ok: undeferred, but not replayable.
+#pragma omp taskgraph
+ {
+#pragma omp task if (0) replayable(0)
+ {}
+ }
+}
+
+// A bare replayable clause, and one whose argument is not a constant, both
+// leave the construct replayable.
+void replayable_forms(int cond) {
+ omp_event_handle_t ev;
+
+#pragma omp taskgraph
+ {
+#pragma omp task detach(ev) replayable // expected-error {{detachable replayable task is not allowed within '#pragma omp taskgraph'}}
+ {}
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp task detach(ev) replayable(1) // expected-error {{detachable replayable task is not allowed within '#pragma omp taskgraph'}}
+ {}
+ }
+
+ // Not a constant, so it may reach a recording: diagnosed, as in flang.
+#pragma omp taskgraph
+ {
+#pragma omp task detach(ev) replayable(cond) // expected-error {{detachable replayable task is not allowed within '#pragma omp taskgraph'}}
+ {}
+ }
+}
+
+// The body of a generated task is not part of the taskgraph region, so the
+// restrictions do not reach into it.
+void nested() {
+ omp_event_handle_t ev;
+
+#pragma omp taskgraph
+ {
+#pragma omp task
+ {
+#pragma omp task detach(ev)
+ {}
+ }
+ }
+}
+
+template <int N> void templated() {
+ omp_event_handle_t ev;
+
+#pragma omp taskgraph
+ {
+#pragma omp task detach(ev) replayable(N) // expected-error {{detachable replayable task is not allowed within '#pragma omp taskgraph'}}
+ {}
+ }
+}
+
+void instantiate() {
+ templated<1>(); // expected-note {{in instantiation of function template specialization 'templated<1>' requested here}}
+ templated<0>();
+}
diff --git a/clang/test/OpenMP/taskgraph_target_replayable_codegen.cpp b/clang/test/OpenMP/taskgraph_target_replayable_codegen.cpp
new file mode 100644
index 00000000000000..9dbb0918b38811
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_target_replayable_codegen.cpp
@@ -0,0 +1,126 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-linux-gnu -fopenmp-targets=x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// Verifies that a 'replayable' clause routes a target / target-data construct
+// to the __kmpc_taskgraph_target* entry points even when the construct is not
+// lexically nested inside an 'omp taskgraph', mirroring how 'omp task' handles
+// the clause:
+// * replayable with no argument (constant-true): emitIfClause folds away the
+// ordinary path, leaving a single taskgraph recording call.
+// * replayable(expr): emitIfClause emits a runtime branch -- the taskgraph
+// recording call when the condition holds, the ordinary launch otherwise.
+
+void replayable_const() {
+ int x = 0;
+#pragma omp target map(tofrom : x) replayable
+ { x++; }
+}
+
+void replayable_cond(int c) {
+ int x = 0;
+#pragma omp target map(tofrom : x) replayable(c)
+ { x++; }
+}
+
+void replayable_enter_data_const() {
+ int x = 0;
+#pragma omp target enter data map(to : x) replayable
+}
+
+void replayable_update_cond(int c) {
+ int x = 0;
+#pragma omp target update to(x) replayable(c)
+}
+
+// A replayable target with a depend clause must forward the dependences to
+// __kmpc_taskgraph_target itself (the entry point satisfies them
+// synchronously) and must NOT be wrapped in a hidden helper task: the recording
+// path executes synchronously.
+void replayable_depend_const() {
+ int x = 0, d = 0;
+#pragma omp target map(tofrom : x) depend(inout : d) replayable
+ { x++; }
+}
+
+// A replayable target whose region is a teams-distribute construct: the
+// recording path evaluates the launch bounds (number of teams, thread limit and
+// the loop trip count) just as the ordinary path does, and those expressions
+// refer to `n`, which is captured by the target region. So the recording path
+// has to emit them inside an inlined region with CapturedStmtInfo installed;
+// emitting them in the enclosing function's context instead used to assert in
+// CodeGenFunction::EmitDeclRefLValue.
+void replayable_teams_distribute(double *p, int n) {
+#pragma omp target replayable
+ {
+#pragma omp teams distribute
+ for (int i = 0; i < n; ++i)
+ p[i] = 0.0;
+ }
+}
+
+// 'replayable' has none of the innermost-leaf, outermost-leaf, all-privatizing
+// or once-for-all-constituents clause properties, so it takes the default
+// all-constituents property and applies to every leaf construct of a compound
+// directive that accepts it. For a compound directive in the target family
+// that is the 'target' leaf alone, so the clause is accepted here and routes
+// the construct to __kmpc_taskgraph_target exactly as it does on a lone
+// 'target'.
+void replayable_combined(double *p, int n) {
+#pragma omp target teams replayable map(tofrom : p[0 : n])
+ { p[0] = 1.0; }
+#pragma omp target teams distribute parallel for replayable map(tofrom : p[0 : n])
+ for (int i = 0; i < n; ++i)
+ p[i] = 0.0;
+}
+
+// Constant-true replayable target: only the taskgraph recording call is
+// emitted; the ordinary __tgt_target_kernel launch is folded away.
+// CHECK-LABEL: define {{.*}}@_Z16replayable_constv
+// CHECK: call i32 @__kmpc_taskgraph_target(
+// CHECK-NOT: @__tgt_target_kernel
+
+// replayable(c): a runtime branch selects between the taskgraph recording call
+// and the ordinary launch.
+// CHECK-LABEL: define {{.*}}@_Z15replayable_condi
+// CHECK: br i1
+// CHECK-DAG: call i32 @__kmpc_taskgraph_target(
+// CHECK-DAG: call i32 @__tgt_target_kernel(
+
+// Constant-true replayable 'target enter data': only the taskgraph recording
+// call is emitted.
+// CHECK-LABEL: define {{.*}}@_Z27replayable_enter_data_constv
+// CHECK: call void @__kmpc_taskgraph_target_enter_data(
+// CHECK-NOT: @__tgt_target_data_begin_mapper
+
+// replayable(c) 'target update': a runtime branch selects between the taskgraph
+// recording call and the ordinary mapper call.
+// CHECK-LABEL: define {{.*}}@_Z22replayable_update_condi
+// CHECK: br i1
+// CHECK-DAG: call void @__kmpc_taskgraph_target_update(
+// CHECK-DAG: call void @__tgt_target_data_update_mapper(
+
+// Constant-true replayable target with a depend clause: the dependences are
+// passed to __kmpc_taskgraph_target (non-zero ndeps + a depend list) and no
+// hidden helper task is created.
+// CHECK-LABEL: define {{.*}}@_Z23replayable_depend_constv
+// CHECK-NOT: @__kmpc_omp_task_alloc
+// CHECK: call i32 @__kmpc_taskgraph_target(ptr {{[^,]*}}, i32 {{[^,]*}}, i32 1, ptr
+// CHECK-NOT: @__kmpc_omp_task_alloc
+
+// Replayable target around a teams-distribute region: still just the taskgraph
+// recording call, with the launch bounds computed for it.
+// CHECK-LABEL: define {{.*}}@_Z27replayable_teams_distributePdi
+// CHECK: call i32 @__kmpc_taskgraph_target(
+// CHECK-NOT: @__tgt_target_kernel
+
+// 'replayable' on a compound directive of the target family applies to the
+// 'target' leaf, so both constructs record rather than launch.
+// CHECK-LABEL: define {{.*}}@_Z19replayable_combinedPdi
+// CHECK: call i32 @__kmpc_taskgraph_target(
+// CHECK: call i32 @__kmpc_taskgraph_target(
+// CHECK-NOT: @__tgt_target_kernel
+
+#endif
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 1b53d4f2a5d754..1bc38a5d4d11f3 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2436,6 +2436,10 @@ void OMPClauseEnqueue::VisitOMPNowaitClause(const OMPNowaitClause *C) {
Visitor->AddStmt(C->getCondition());
}
+void OMPClauseEnqueue::VisitOMPReplayableClause(const OMPReplayableClause *C) {
+ Visitor->AddStmt(C->getCondition());
+}
+
void OMPClauseEnqueue::VisitOMPUntiedClause(const OMPUntiedClause *) {}
void OMPClauseEnqueue::VisitOMPMergeableClause(const OMPMergeableClause *) {}
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index e6fbee3dd9c1f0..393efc9344d269 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -510,6 +510,7 @@ def OMPC_Release : Clause<[Spelling<"release">]> {
let clangClass = "OMPReleaseClause";
}
def OMPC_Replayable : Clause<[Spelling<"replayable">]> {
+ let clangClass = "OMPReplayableClause";
let flangClass = "OmpReplayableClause";
let isValueOptional = true;
}
@@ -1797,6 +1798,7 @@ def OMP_target_loop : Directive<[Spelling<"target loop">]> {
VersionedClause<OMPC_NoWait>,
VersionedClause<OMPC_OMPX_DynCGroupMem>,
VersionedClause<OMPC_Order>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_ThreadLimit>,
];
let leafConstructs = [OMP_Target, OMP_loop];
@@ -2308,6 +2310,7 @@ def OMP_TargetParallel : Directive<[Spelling<"target parallel">]> {
VersionedClause<OMPC_NumThreads>,
VersionedClause<OMPC_OMPX_DynCGroupMem>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_Severity, 60>,
VersionedClause<OMPC_ThreadLimit, 51>,
];
@@ -2342,6 +2345,7 @@ def OMP_TargetParallelDo : Directive<[Spelling<"target parallel do">]> {
VersionedClause<OMPC_Order, 50>,
VersionedClause<OMPC_Ordered>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_Schedule>,
VersionedClause<OMPC_Severity, 60>,
];
@@ -2380,6 +2384,7 @@ def OMP_TargetParallelDoSimd
VersionedClause<OMPC_Order, 50>,
VersionedClause<OMPC_Ordered>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_SafeLen>,
VersionedClause<OMPC_Schedule>,
VersionedClause<OMPC_Severity, 60>,
@@ -2419,6 +2424,7 @@ def OMP_TargetParallelFor : Directive<[Spelling<"target parallel for">]> {
VersionedClause<OMPC_Order, 50>,
VersionedClause<OMPC_Ordered>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_Schedule>,
VersionedClause<OMPC_Severity, 60>,
VersionedClause<OMPC_ThreadLimit, 51>,
@@ -2460,6 +2466,7 @@ def OMP_TargetParallelForSimd
VersionedClause<OMPC_Order, 50>,
VersionedClause<OMPC_Ordered>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_SafeLen>,
VersionedClause<OMPC_Schedule>,
VersionedClause<OMPC_Severity, 60>,
@@ -2499,6 +2506,7 @@ def OMP_target_parallel_loop : Directive<[Spelling<"target parallel loop">]> {
VersionedClause<OMPC_OMPX_DynCGroupMem>,
VersionedClause<OMPC_Order>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_Severity, 60>,
VersionedClause<OMPC_ThreadLimit, 51>,
];
@@ -2535,6 +2543,7 @@ def OMP_TargetSimd : Directive<[Spelling<"target simd">]> {
VersionedClause<OMPC_OMPX_DynCGroupMem>,
VersionedClause<OMPC_Order, 50>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_SafeLen>,
VersionedClause<OMPC_Schedule>,
VersionedClause<OMPC_Severity, 60>,
@@ -2568,6 +2577,7 @@ def OMP_TargetTeams : Directive<[Spelling<"target teams">]> {
VersionedClause<OMPC_NumTeams>,
VersionedClause<OMPC_OMPX_Bare>,
VersionedClause<OMPC_OMPX_DynCGroupMem>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_ThreadLimit>,
];
let leafConstructs = [OMP_Target, OMP_Teams];
@@ -2601,6 +2611,7 @@ def OMP_TargetTeamsDistribute
VersionedClause<OMPC_NumTeams>,
VersionedClause<OMPC_OMPX_DynCGroupMem>,
VersionedClause<OMPC_Order, 50>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_ThreadLimit>,
];
let leafConstructs = [OMP_Target, OMP_Teams, OMP_Distribute];
@@ -2636,6 +2647,7 @@ def OMP_TargetTeamsDistributeParallelDo
VersionedClause<OMPC_NumThreads>,
VersionedClause<OMPC_Order, 50>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_Schedule>,
VersionedClause<OMPC_Severity, 60>,
VersionedClause<OMPC_ThreadLimit>,
@@ -2677,6 +2689,7 @@ def OMP_TargetTeamsDistributeParallelDoSimd
VersionedClause<OMPC_NumThreads>,
VersionedClause<OMPC_Order, 50>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_SafeLen>,
VersionedClause<OMPC_Schedule>,
VersionedClause<OMPC_Severity, 60>,
@@ -2719,6 +2732,7 @@ def OMP_TargetTeamsDistributeParallelFor
VersionedClause<OMPC_OMPX_DynCGroupMem>,
VersionedClause<OMPC_Order, 50>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_Schedule>,
VersionedClause<OMPC_Severity, 60>,
VersionedClause<OMPC_ThreadLimit>,
@@ -2762,6 +2776,7 @@ def OMP_TargetTeamsDistributeParallelForSimd
VersionedClause<OMPC_OMPX_DynCGroupMem>,
VersionedClause<OMPC_Order, 50>,
VersionedClause<OMPC_ProcBind>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_SafeLen>,
VersionedClause<OMPC_Schedule>,
VersionedClause<OMPC_Severity, 60>,
@@ -2804,6 +2819,7 @@ def OMP_TargetTeamsDistributeSimd
VersionedClause<OMPC_NumTeams>,
VersionedClause<OMPC_OMPX_DynCGroupMem>,
VersionedClause<OMPC_Order, 50>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_SafeLen>,
VersionedClause<OMPC_SimdLen>,
VersionedClause<OMPC_ThreadLimit>,
@@ -2834,6 +2850,7 @@ def OMP_TargetTeamsWorkdistribute : Directive<[Spelling<"target teams workdistri
VersionedClause<OMPC_NumTeams>,
VersionedClause<OMPC_OMPX_Bare>,
VersionedClause<OMPC_OMPX_DynCGroupMem>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_ThreadLimit>,
];
let leafConstructs = [OMP_Target, OMP_Teams, OMP_Workdistribute];
@@ -2867,6 +2884,7 @@ def OMP_target_teams_loop : Directive<[Spelling<"target teams loop">]> {
VersionedClause<OMPC_NumTeams>,
VersionedClause<OMPC_OMPX_DynCGroupMem>,
VersionedClause<OMPC_Order>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_ThreadLimit>,
];
let leafConstructs = [OMP_Target, OMP_Teams, OMP_loop];
diff --git a/openmp/runtime/test/taskgraph/taskgraph_detach_dynamic_fails.cpp b/openmp/runtime/test/taskgraph/taskgraph_detach_dynamic_fails.cpp
new file mode 100644
index 00000000000000..5ac39f399eef6e
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_detach_dynamic_fails.cpp
@@ -0,0 +1,52 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %not --crash %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// OpenMP 6.0 [14.3]: a detachable task must not be a replayable task in a
+// taskgraph region. The completion event is created when the task is
+// generated, and a replay does not generate the task again, so there is no
+// event left for the program to fulfil: the node never completes and the
+// encounter waits on its taskgroup for ever. That is what this used to do.
+//
+// Clang rejects the lexically-nested form outright (see
+// clang/test/OpenMP/taskgraph_replayable_restrictions.cpp). This is the form
+// it cannot see: the construct is in a function called from the region, so
+// whether it is recorded is only known once the call has been made.
+
+#include <cstdio>
+#include <omp.h>
+
+static void make_task(int replayable) {
+ omp_event_handle_t ev;
+#pragma omp task detach(ev) replayable(replayable)
+ {
+ }
+ omp_fulfill_event(ev);
+}
+
+int main() {
+#pragma omp parallel num_threads(2)
+#pragma omp single
+ {
+ // Detachable but not replayable: recorded as nothing, runs normally, and
+ // must be left alone.
+ for (int rep = 0; rep < 2; ++rep) {
+#pragma omp taskgraph graph_id(1)
+ make_task(0);
+ }
+ std::fprintf(stderr, "not replayable: ok\n");
+
+ // Replayable: has to be rejected rather than recorded.
+#pragma omp taskgraph graph_id(2)
+ make_task(1);
+ }
+
+ std::fprintf(stderr, "UNEXPECTED SUCCESS\n");
+ return 0;
+}
+
+// CHECK: not replayable: ok
+// CHECK: OMP: Error #{{[0-9]+}}: {{.*}}a detachable task cannot be
+// CHECK-SAME: recorded for taskgraph replay
+// CHECK-NOT: UNEXPECTED SUCCESS
diff --git a/openmp/runtime/test/taskgraph/taskgraph_nested_replayable.cpp b/openmp/runtime/test/taskgraph/taskgraph_nested_replayable.cpp
new file mode 100644
index 00000000000000..cd277356fec419
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_nested_replayable.cpp
@@ -0,0 +1,47 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A task with a 'replayable' clause in the body of a task that the taskgraph
+// records. The inner construct is encountered by the outer task, whose
+// region is not part of the taskgraph region (OpenMP 6.0 [2], region), so it
+// is not a replayable construct of the region and is not recorded: a replay
+// runs the outer task's body again, which generates the inner task once, from
+// the values of that replay.
+
+#include <cstdio>
+
+static int Base;
+static int Runs;
+static int Seen;
+
+int main() {
+#pragma omp parallel num_threads(4)
+#pragma omp single
+ for (int R = 0; R < 4; ++R) {
+ Base = R * 10;
+ Runs = 0;
+#pragma omp taskgraph
+ {
+#pragma omp task shared(Runs, Seen)
+ {
+ int W = Base + 1;
+#pragma omp task replayable firstprivate(W) shared(Runs, Seen)
+ {
+#pragma omp atomic
+ ++Runs;
+ Seen = W;
+ }
+#pragma omp taskwait
+ }
+ }
+ std::printf("run %d: inner ran %d time(s), saw %d\n", R, Runs, Seen);
+ }
+ return 0;
+}
+
+// CHECK: run 0: inner ran 1 time(s), saw 1
+// CHECK-NEXT: run 1: inner ran 1 time(s), saw 11
+// CHECK-NEXT: run 2: inner ran 1 time(s), saw 21
+// CHECK-NEXT: run 3: inner ran 1 time(s), saw 31
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_false.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_false.cpp
new file mode 100644
index 00000000000000..bbe8ad9e7bcf6a
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_false.cpp
@@ -0,0 +1,141 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// OpenMP 6.0 [14.3]: a task-generating construct encountered in a taskgraph
+// construct is a replayable construct of the region "unless otherwise specified
+// by the replayable clause", and "a replay execution does not entail execution
+// of any code that is part of both the taskgraph region and the encountering
+// task region". Together those mean a replayable(false) construct written
+// inside a taskgraph region is encountered once, on the execution that runs the
+// region body, and is absent from every replay -- while its replayable
+// neighbours run on every execution.
+//
+// This is the behavioural half of
+// clang/test/OpenMP/taskgraph_replayable_false_codegen.cpp, which pins which
+// runtime entry point each construct is routed to.
+
+#include <atomic>
+#include <cstdio>
+
+static constexpr int NumReps = 4; // one recording execution + three replays
+static constexpr int LoopTrips = 5;
+
+static std::atomic<int> Failures{0};
+
+// The dependence-ordering case below wants a variable that is shared, not
+// firstprivate by default, in the tasks that carry the depend clauses, and one
+// that the taskgraph region does not have to save a copy of: static storage
+// duration gives both.
+static int DepVar;
+static std::atomic<int> OrderViolations{0};
+
+static void expect(const char *What, int Actual, int Expected) {
+ if (Actual != Expected) {
+ std::fprintf(stderr, "FAIL %s=%d expected=%d\n", What, Actual, Expected);
+ ++Failures;
+ }
+}
+
+// Enough work to make an unsynchronised read of Ordered likely to lose the
+// race, so that the taskwait case below is not vacuous.
+static void spin() {
+ volatile int sink = 0;
+ for (int i = 0; i < 2000000; ++i)
+ sink += i;
+}
+
+int main() {
+ std::atomic<int> Replayed{0}, Once{0};
+ std::atomic<int> LoopReplayed{0}, LoopOnce{0};
+ std::atomic<int> ResetEveryTime{0};
+ std::atomic<int> DynFalseAtRecord{0}, Chained{0};
+
+#pragma omp parallel num_threads(4)
+#pragma omp single
+ {
+ for (int rep = 0; rep < NumReps; ++rep) {
+#pragma omp taskgraph graph_id(1)
+ {
+#pragma omp task
+ ++Replayed;
+#pragma omp task replayable(false)
+ ++Once;
+#pragma omp taskloop
+ for (int i = 0; i < LoopTrips; ++i)
+ ++LoopReplayed;
+#pragma omp taskloop replayable(false)
+ for (int i = 0; i < LoopTrips; ++i)
+ ++LoopOnce;
+ }
+ }
+
+ // Re-recording the graph on every encounter runs the body every time, so
+ // the non-replayable task is encountered every time too. Without this the
+ // "runs once" result above could just as well be a task that never ran
+ // more than once for some unrelated reason.
+ for (int rep = 0; rep < NumReps; ++rep) {
+#pragma omp taskgraph graph_id(2) graph_reset
+ {
+#pragma omp task replayable(false)
+ ++ResetEveryTime;
+ }
+ }
+
+ // A non-constant argument is decided when the construct is encountered,
+ // which only happens on the execution that runs the body. So a condition
+ // that is false for the recording execution keeps the construct out of the
+ // record for good, whatever it would evaluate to later.
+ for (int rep = 0; rep < NumReps; ++rep) {
+ const int Replay = rep != 0;
+#pragma omp taskgraph graph_id(3)
+ {
+#pragma omp task replayable(Replay)
+ ++DynFalseAtRecord;
+ }
+ }
+
+ // Dependences between two non-replayable constructs are what the November
+ // 2025 errata permits (both sides not replayable), and on the recording
+ // execution they are ordered by the ordinary dependence machinery.
+ for (int rep = 0; rep < NumReps; ++rep) {
+ DepVar = 0;
+#pragma omp taskgraph graph_id(4)
+ {
+#pragma omp task depend(out : DepVar) replayable(false)
+ {
+ spin();
+ DepVar = 42;
+ }
+#pragma omp task depend(in : DepVar) replayable(false)
+ {
+ if (DepVar != 42)
+ ++OrderViolations;
+ ++Chained;
+ }
+ // A non-replayable taskwait has to wait for real on this execution.
+#pragma omp taskwait depend(in : DepVar) replayable(false)
+ if (DepVar != 42)
+ ++OrderViolations;
+ }
+ }
+ }
+
+ expect("replayed", Replayed.load(), NumReps);
+ expect("once", Once.load(), 1);
+ expect("loop_replayed", LoopReplayed.load(), NumReps * LoopTrips);
+ expect("loop_once", LoopOnce.load(), LoopTrips);
+ expect("reset_every_time", ResetEveryTime.load(), NumReps);
+ expect("dyn_false_at_record", DynFalseAtRecord.load(), 1);
+ expect("chained", Chained.load(), 1);
+ expect("order_violations", OrderViolations.load(), 0);
+
+ if (Failures.load())
+ return 1;
+ std::fprintf(stderr, "PASS replayable(false)\n");
+ return 0;
+}
+
+// CHECK-NOT: FAIL
+// CHECK: PASS replayable(false)
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_false_reduction.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_false_reduction.cpp
new file mode 100644
index 00000000000000..45a8d01119e852
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_false_reduction.cpp
@@ -0,0 +1,88 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A reduction on a construct that turns out not to be a replayable construct.
+//
+// While a taskgraph is recording, the reduction init hands its input data to
+// the runtime to stash on the taskgroup, so that the next node recorded in that
+// taskgroup can adopt it and re-create the reduction state on each replay. A
+// construct that is not replayable records no node, so nothing adopts the
+// stash. Two cases, and they are handled in different places:
+//
+// - replayable(false): the compiler can see the construct will take the
+// ordinary path, so it routes the reduction to the plain init and no stash
+// is created at all.
+// - replayable(non-constant): the compiler cannot, so the stash is created
+// and the runtime discards it when the taskgroup ends. This used to trip
+// an assertion there (and leak in a build without assertions).
+//
+// Either way the reduction itself must produce the right answer on the
+// execution that runs the region body, and the taskloop must be absent from
+// every replay.
+
+#include <atomic>
+#include <cstdio>
+
+static constexpr int NumReps = 3; // one recording execution + two replays
+static constexpr int LoopTrips = 16;
+
+static volatile int Sum;
+static std::atomic<int> Failures{0};
+
+static int expected_sum() {
+ int Sum = 0;
+ for (int i = 0; i < LoopTrips; ++i)
+ Sum += i;
+ return Sum;
+}
+
+static void expect(const char *What, int Rep, int Actual, int Expected) {
+ if (Actual != Expected) {
+ std::fprintf(stderr, "FAIL %s rep=%d got=%d expected=%d\n", What, Rep,
+ Actual, Expected);
+ ++Failures;
+ }
+}
+
+int main() {
+ const int Expected = expected_sum();
+ int NotReplayable = 0;
+
+#pragma omp parallel num_threads(4)
+#pragma omp single
+ {
+ for (int rep = 0; rep < NumReps; ++rep) {
+ Sum = 0;
+#pragma omp taskgraph graph_id(1)
+ {
+#pragma omp taskloop replayable(false) num_tasks(4) reduction(+ : Sum)
+ for (int i = 0; i < LoopTrips; ++i)
+ Sum += i;
+ }
+ // The body only runs on the recording execution, so only that one
+ // reduces; a replay has no taskloop to run and leaves Sum alone.
+ expect("constant", rep, Sum, rep == 0 ? Expected : 0);
+ }
+
+ for (int rep = 0; rep < NumReps; ++rep) {
+ Sum = 0;
+#pragma omp taskgraph graph_id(2)
+ {
+#pragma omp taskloop replayable(NotReplayable) num_tasks(4) reduction(+ : Sum)
+ for (int i = 0; i < LoopTrips; ++i)
+ Sum += i;
+ }
+ expect("dynamic", rep, Sum, rep == 0 ? Expected : 0);
+ }
+ }
+
+ if (Failures.load())
+ return 1;
+ std::fprintf(stderr, "PASS replayable(false) reduction sum=%d\n", Expected);
+ return 0;
+}
+
+// CHECK-NOT: FAIL
+// CHECK: PASS replayable(false) reduction sum=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_recursive_reduction.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_recursive_reduction.cpp
new file mode 100644
index 00000000000000..c349bd2f34146c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_recursive_reduction.cpp
@@ -0,0 +1,147 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Exercises reduction(+: ...) on dynamically (non-lexically) nested
+// replayable constructs reached through a recursive driver:
+//
+// - emit_reduction_taskloop(): replayable taskloop performing a reduction
+// into a file-scope accumulator (Sum). The
+// 'seed' parameter is captured as
+// firstprivate(saved:) so the snapshot
+// taken at recording is reused unchanged
+// on every replay (OpenMP 6.0 [7.5.4] /
+// [14.3]).
+// - emit_publish_task(): replayable task that reads Sum and stores
+// it into a distinct slot of a file-scope
+// Snapshots[] array; the slot index is
+// carried into the task as a
+// firstprivate(saved:) capture so that
+// every recursion-level publish lands in
+// its own slot and no two publishes race
+// for the same destination
+// - driver(): recursive function that calls
+// emit_reduction_taskloop() on descent and
+// emit_publish_task() on ascent, so on
+// replay these constructs are dispatched
+// from differing recursive stack frames
+// - run_taskgraph(): wraps the recursive driver() in
+// #pragma omp taskgraph and returns Sum
+// (the cumulative reduction, which is the
+// deterministic test signal; Snapshots[]
+// is written purely as a side-effect to
+// exercise the replayable publish tasks)
+//
+// Per the saved-snapshot semantics, every replay reproduces the recording
+// run's reduction value, independent of the seed argument passed to the
+// replay invocation. The test therefore (i) verifies that the recording
+// matches expected_result(Seeds[0]) and (ii) verifies that every replay
+// matches the recording. Historically this exercise hit two consecutive
+// crashes on replay: first a relocate-side #302 because the (static)
+// reduction target's shareds slot couldn't be re-projected from a
+// non-OpenMP outer scope, then a taskred-lookup assertion because the
+// recording-time taskgroup_t holding the reduction state had been torn
+// down. Both gaps are now fixed in the compiler and runtime.
+
+#include <cstdio>
+
+static constexpr int MaxDepth = 4;
+
+static volatile int Sum = 0;
+static volatile int Snapshots[MaxDepth] = {0, 0, 0, 0};
+
+__attribute__((noinline)) static void emit_reduction_taskloop(int seed) {
+#pragma omp taskloop replayable num_tasks(8) reduction(+ : Sum) \
+ firstprivate(saved : seed)
+ for (int i = 0; i < 16; ++i)
+ Sum += seed + i;
+}
+
+__attribute__((noinline)) static void emit_publish_task(int slot) {
+#pragma omp task replayable firstprivate(saved : slot)
+ {
+ Snapshots[slot] = Sum;
+ }
+}
+
+__attribute__((noinline)) static void driver(int depth, int seed) {
+ emit_reduction_taskloop(seed + depth);
+ if (depth == 0) {
+ emit_publish_task(depth);
+ return;
+ }
+ driver(depth - 1, seed);
+ emit_publish_task(depth);
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+ Sum = 0;
+ for (int i = 0; i < MaxDepth; ++i)
+ Snapshots[i] = 0;
+
+#pragma omp taskgraph graph_id(917)
+ {
+ driver(MaxDepth - 1, seed);
+ }
+
+ // Sum is the deterministic test signal: after the taskgraph's implicit
+ // taskwait it holds the cumulative reduction across all MaxDepth
+ // taskloops. The per-slot Snapshots[] entries are written by the
+ // replayable publish tasks but race with the reduction taskloops, so
+ // their individual values are non-deterministic and are intentionally
+ // not compared here. The array exists only so that each replayable
+ // publish task targets a distinct destination instead of clobbering a
+ // shared scalar.
+ return Sum;
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+ int sum = 0;
+ for (int d = MaxDepth - 1; d >= 0; --d)
+ for (int i = 0; i < 16; ++i)
+ sum += seed + d + i;
+ return sum;
+}
+
+int main() {
+ constexpr int NumRuns = 4;
+ constexpr int Seeds[NumRuns] = {1, 5, 11, 23};
+
+ int recorded = -1;
+ bool failed = false;
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ recorded = run_taskgraph(Seeds[0]);
+ const int exp0 = expected_result(Seeds[0]);
+ if (recorded != exp0) {
+ std::fprintf(stderr, "FAIL initial record got=%d expected=%d\n",
+ recorded, exp0);
+ failed = true;
+ }
+
+ // Saved-snapshot semantics: every replay should reproduce the
+ // recording's reduction value, regardless of the live argument.
+ for (int i = 1; i < NumRuns; ++i) {
+ const int replayed = run_taskgraph(Seeds[i]);
+ if (replayed != recorded) {
+ std::fprintf(stderr, "FAIL replay %d seed=%d got=%d recorded=%d\n", i,
+ Seeds[i], replayed, recorded);
+ failed = true;
+ }
+ }
+ }
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr, "PASS non-lexical recursive reduction result=%d\n",
+ recorded);
+ return 0;
+}
+
+// CHECK: PASS non-lexical recursive reduction result=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_reduction_minimal_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_reduction_minimal_taskloop.cpp
new file mode 100644
index 00000000000000..17b36f16a02ef5
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_reduction_minimal_taskloop.cpp
@@ -0,0 +1,105 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Minimal end-to-end exercise of reduction(+: ...) on a replayable taskloop
+// that is dynamically (non-lexically) nested inside a taskgraph: the
+// taskloop directive lives in a helper called from inside the taskgraph
+// region, not in the taskgraph's lexical body.
+//
+// - Sum is a file-scope static, so the reduction's accumulator address is
+// stable across runs and the taskgraph relocate helper has no shareds
+// slot it must refresh (the static capture is link-time-fixed).
+// - 'seed' is captured into the taskloop's per-task '.kmp_privates.t'
+// snapshot as firstprivate(saved: ...); per OpenMP 6.0 [7.5.4] / [14.3]
+// the snapshot is taken at recording time and reused unchanged on every
+// replay.
+// - The recording run therefore produces the expected reduction for
+// Seeds[0], and every subsequent replay (each invoked with a different
+// seed) produces exactly the recorded value because the saved snapshot
+// of 'seed' is what the body sees.
+//
+// Historically this test crashed on replay with OMP: Error #302 because
+// the taskgraph relocate helper refused to handle the (static) reduction
+// capture, and even after that was fixed the reduction body still
+// asserted because the recording-time taskgroup_t had been torn down and
+// the taskred state was unreachable. Both gaps are addressed: the
+// relocate helper now treats captures of static-storage variables as
+// no-op-safe, and the runtime now stashes reduction-init input into the
+// surrounding taskgraph at recording so the replay machinery can re-create
+// the taskred state on every replay.
+
+#include <cstdio>
+
+static volatile int Sum = 0;
+
+__attribute__((noinline)) static void emit_reduction_taskloop(int seed) {
+#pragma omp taskloop replayable num_tasks(8) reduction(+ : Sum) \
+ firstprivate(saved : seed)
+ for (int i = 0; i < 16; ++i)
+ Sum += seed + i;
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+ Sum = 0;
+
+#pragma omp taskgraph graph_id(921)
+ {
+ emit_reduction_taskloop(seed);
+ }
+
+ return Sum;
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+ int sum = 0;
+ for (int i = 0; i < 16; ++i)
+ sum += seed + i;
+ return sum;
+}
+
+int main() {
+ constexpr int NumRuns = 4;
+ constexpr int Seeds[NumRuns] = {1, 5, 11, 23};
+
+ int recorded = -1;
+ bool failed = false;
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ recorded = run_taskgraph(Seeds[0]);
+ const int exp0 = expected_result(Seeds[0]);
+ if (recorded != exp0) {
+ std::fprintf(stderr, "FAIL initial record got=%d expected=%d\n",
+ recorded, exp0);
+ failed = true;
+ }
+
+ // 'seed' is firstprivate(saved:), so every replay's body sees the
+ // snapshot taken at recording time. The reduction therefore yields
+ // the same value as the recording regardless of the live argument
+ // passed on the replay.
+ for (int i = 1; i < NumRuns; ++i) {
+ const int replayed = run_taskgraph(Seeds[i]);
+ if (replayed != recorded) {
+ std::fprintf(stderr, "FAIL replay %d seed=%d got=%d recorded=%d\n", i,
+ Seeds[i], replayed, recorded);
+ failed = true;
+ }
+ }
+ }
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr,
+ "PASS non-lexical replayable taskloop reduction result=%d\n",
+ recorded);
+ return 0;
+}
+
+// CHECK: PASS non-lexical replayable taskloop reduction result=
>From dbe5a0e49b7ac2b3f9cc44b2f0fbf607618e1b72 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 23 Apr 2026 14:47:24 -0500
Subject: [PATCH 07/29] [OpenMP] Add taskgraph tests adapted from the previous
implementation
This patch contains new tests from previous iterations of taskgraph
support, with some changes to adjust to the new implementation.
Co-Authored-By: Josep Pinot <josep.pinot at bsc.es>
Pull Request: https://github.com/llvm/llvm-project/pull/194055
---
openmp/runtime/test/tasking/omp_taskgraph.cpp | 37 ++++++++++
.../test/tasking/omp_taskgraph_deps.cpp | 54 +++++++++++++++
.../test/tasking/omp_taskgraph_multiTDGs.cpp | 68 +++++++++++++++++++
.../test/tasking/omp_taskgraph_taskloop.cpp | 41 +++++++++++
4 files changed, 200 insertions(+)
create mode 100644 openmp/runtime/test/tasking/omp_taskgraph.cpp
create mode 100644 openmp/runtime/test/tasking/omp_taskgraph_deps.cpp
create mode 100644 openmp/runtime/test/tasking/omp_taskgraph_multiTDGs.cpp
create mode 100644 openmp/runtime/test/tasking/omp_taskgraph_taskloop.cpp
diff --git a/openmp/runtime/test/tasking/omp_taskgraph.cpp b/openmp/runtime/test/tasking/omp_taskgraph.cpp
new file mode 100644
index 00000000000000..67567a9b31870f
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_taskgraph.cpp
@@ -0,0 +1,37 @@
+// clang-format off
+// REQUIRES: omp_taskgraph_experimental
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run
+// clang-format on
+#include <iostream>
+#include <cassert>
+#define NT 100
+
+// Compiler-generated code (emulation)
+typedef struct ident {
+ void *dummy;
+} ident_t;
+
+void func(int *num_exec) { (*num_exec)++; }
+
+int main() {
+ int num_exec = 0;
+ int num_tasks = 0;
+ int x = 0;
+#pragma omp parallel
+#pragma omp single
+ for (int iter = 0; iter < NT; ++iter) {
+#pragma omp taskgraph
+ {
+ num_tasks++;
+#pragma omp task
+ func(&num_exec);
+ }
+ }
+
+ assert(num_tasks == 1);
+ assert(num_exec == NT);
+
+ std::cout << "Passed" << std::endl;
+ return 0;
+}
+// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_taskgraph_deps.cpp b/openmp/runtime/test/tasking/omp_taskgraph_deps.cpp
new file mode 100644
index 00000000000000..e83e0c2f2ce620
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_taskgraph_deps.cpp
@@ -0,0 +1,54 @@
+// clang-format off
+// REQUIRES: omp_taskgraph_experimental
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run
+// clang-format on
+#include <iostream>
+#include <cassert>
+#define NT 100
+#define MULTIPLIER 100
+#define DECREMENT 5
+
+int val;
+// Compiler-generated code (emulation)
+typedef struct ident {
+ void *dummy;
+} ident_t;
+
+void sub() {
+#pragma omp atomic
+ val -= DECREMENT;
+}
+
+void add() {
+#pragma omp atomic
+ val += DECREMENT;
+}
+
+void mult() {
+ // no atomicity needed, can only be executed by 1 thread
+ // and no concurrency with other tasks possible
+ val *= MULTIPLIER;
+}
+
+int main() {
+ val = 0;
+ int *x, *y;
+#pragma omp parallel
+#pragma omp single
+ for (int iter = 0; iter < NT; ++iter) {
+#pragma omp taskgraph
+ {
+#pragma omp task depend(out : y)
+ add();
+#pragma omp task depend(out : x)
+ sub();
+#pragma omp task depend(in : x, y)
+ mult();
+ }
+ }
+ assert(val == 0);
+
+ std::cout << "Passed" << std::endl;
+ return 0;
+}
+// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_taskgraph_multiTDGs.cpp b/openmp/runtime/test/tasking/omp_taskgraph_multiTDGs.cpp
new file mode 100644
index 00000000000000..eedd4e642c9565
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_taskgraph_multiTDGs.cpp
@@ -0,0 +1,68 @@
+// clang-format off
+// REQUIRES: omp_taskgraph_experimental
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run
+// clang-format on
+#include <iostream>
+#include <cassert>
+#define NT 20
+#define MULTIPLIER 100
+#define DECREMENT 5
+
+// Compiler-generated code (emulation)
+typedef struct ident {
+ void *dummy;
+} ident_t;
+
+int val;
+
+void sub() {
+#pragma omp atomic
+ val -= DECREMENT;
+}
+
+void add() {
+#pragma omp atomic
+ val += DECREMENT;
+}
+
+void mult() {
+ // no atomicity needed, can only be executed by 1 thread
+ // and no concurrency with other tasks possible
+ val *= MULTIPLIER;
+}
+
+int main() {
+ int num_tasks = 0;
+ int *x, *y;
+#pragma omp parallel
+#pragma omp single
+ for (int iter = 0; iter < NT; ++iter) {
+#pragma omp taskgraph
+ {
+ num_tasks++;
+#pragma omp task depend(out : y)
+ add();
+#pragma omp task depend(out : x)
+ sub();
+#pragma omp task depend(in : x, y)
+ mult();
+ }
+#pragma omp taskgraph
+ {
+ num_tasks++;
+#pragma omp task depend(out : y)
+ add();
+#pragma omp task depend(out : x)
+ sub();
+#pragma omp task depend(in : x, y)
+ mult();
+ }
+ }
+
+ assert(num_tasks == 2);
+ assert(val == 0);
+
+ std::cout << "Passed" << std::endl;
+ return 0;
+}
+// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_taskgraph_taskloop.cpp b/openmp/runtime/test/tasking/omp_taskgraph_taskloop.cpp
new file mode 100644
index 00000000000000..f4fe290a90f0fc
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_taskgraph_taskloop.cpp
@@ -0,0 +1,41 @@
+// clang-format off
+// REQUIRES: omp_taskgraph_experimental
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run
+// clang-format on
+#include <iostream>
+#include <cassert>
+
+#define NT 20
+#define N 128 * 128
+
+typedef struct ident {
+ void *dummy;
+} ident_t;
+
+int main() {
+ int num_tasks = 0;
+
+ int array[N];
+ for (int i = 0; i < N; ++i)
+ array[i] = 1;
+
+ long sum = 0;
+#pragma omp parallel
+#pragma omp single
+ for (int iter = 0; iter < NT; ++iter) {
+#pragma omp taskgraph
+ {
+ num_tasks++;
+#pragma omp taskloop reduction(+ : sum) num_tasks(4096)
+ for (int i = 0; i < N; ++i) {
+ sum += array[i];
+ }
+ }
+ }
+ assert(sum == N * NT);
+ assert(num_tasks == 1);
+
+ std::cout << "Passed" << std::endl;
+ return 0;
+}
+// CHECK: Passed
>From 5c28f04bdc27e4af77956ec0c11434d6f8eb9c34 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 25 Mar 2026 16:57:25 -0500
Subject: [PATCH 08/29] [OpenMP] OpenMP 6.0 "taskgraph" support, add new tests
These are new tests intended to cover the functionality present in the
new taskgraph implementation, including the series-parallel/irreducible
lowering code, the graph_id and graph_reset clauses (with and without an
argument, and during a replay), 'mutexinoutset' dependences (lowered to
'exclusive' regions, or not), irreducible region execution, and a few
other bits.
Assisted-By: Codex
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/194057
---
.../test/taskgraph/taskgraph_deadlock_1.cpp | 55 ++++++++
.../test/taskgraph/taskgraph_deps_1.cpp | 53 ++++++++
.../test/taskgraph/taskgraph_deps_10.cpp | 50 +++++++
.../test/taskgraph/taskgraph_deps_11.cpp | 60 +++++++++
.../test/taskgraph/taskgraph_deps_12.cpp | 55 ++++++++
.../test/taskgraph/taskgraph_deps_13.cpp | 45 +++++++
.../test/taskgraph/taskgraph_deps_14.cpp | 48 +++++++
.../test/taskgraph/taskgraph_deps_15.cpp | 75 +++++++++++
.../test/taskgraph/taskgraph_deps_16.cpp | 55 ++++++++
.../test/taskgraph/taskgraph_deps_17.cpp | 68 ++++++++++
.../test/taskgraph/taskgraph_deps_18.cpp | 46 +++++++
.../test/taskgraph/taskgraph_deps_19.cpp | 51 +++++++
.../test/taskgraph/taskgraph_deps_2.cpp | 58 ++++++++
.../test/taskgraph/taskgraph_deps_20.cpp | 51 +++++++
.../test/taskgraph/taskgraph_deps_21.cpp | 52 ++++++++
.../test/taskgraph/taskgraph_deps_22.cpp | 70 ++++++++++
.../test/taskgraph/taskgraph_deps_23.cpp | 103 ++++++++++++++
.../test/taskgraph/taskgraph_deps_24.cpp | 80 +++++++++++
.../test/taskgraph/taskgraph_deps_25.cpp | 89 +++++++++++++
.../test/taskgraph/taskgraph_deps_26.cpp | 61 +++++++++
.../test/taskgraph/taskgraph_deps_27.cpp | 63 +++++++++
.../test/taskgraph/taskgraph_deps_3.cpp | 62 +++++++++
.../test/taskgraph/taskgraph_deps_4.cpp | 59 ++++++++
.../test/taskgraph/taskgraph_deps_5.cpp | 63 +++++++++
.../test/taskgraph/taskgraph_deps_6.cpp | 59 ++++++++
.../test/taskgraph/taskgraph_deps_7.cpp | 59 ++++++++
.../test/taskgraph/taskgraph_deps_8.cpp | 39 ++++++
.../test/taskgraph/taskgraph_deps_9.cpp | 47 +++++++
...taskgraph_deps_irreducible_graph_reset.cpp | 50 +++++++
.../taskgraph/taskgraph_empty_graph_reset.cpp | 58 ++++++++
.../test/taskgraph/taskgraph_graph_id.cpp | 56 ++++++++
.../taskgraph_graph_id_and_reset.cpp | 57 ++++++++
...taskgraph_graph_id_concurrent_rerecord.cpp | 50 +++++++
.../test/taskgraph/taskgraph_graph_reset.cpp | 57 ++++++++
.../taskgraph_graph_reset_no_argument.cpp | 64 +++++++++
.../taskgraph_if_implicit_taskgroup.cpp | 90 +++++++++++++
...taskgraph_irreducible_nested_exclusive.cpp | 82 ++++++++++++
.../taskgraph_irreducible_nested_parallel.cpp | 87 ++++++++++++
...askgraph_irreducible_nested_sequential.cpp | 75 +++++++++++
.../taskgraph_mutex_exclusive_serialized.cpp | 83 ++++++++++++
.../taskgraph_mutex_irreducible_locks.cpp | 112 ++++++++++++++++
.../taskgraph_mutex_runtime_locks.cpp | 96 +++++++++++++
...askgraph_record_serial_replay_parallel.cpp | 50 +++++++
.../taskgraph/taskgraph_reduction_join.cpp | 53 ++++++++
.../taskgraph_reset_during_replay.cpp | 94 +++++++++++++
.../taskgraph/taskgraph_reset_no_leak.cpp | 126 ++++++++++++++++++
.../test/taskgraph/taskgraph_untied_parts.cpp | 38 ++++++
.../tasking/omp_taskgraph_irreducible.cpp | 84 ++++++++++++
48 files changed, 3138 insertions(+)
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deadlock_1.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_1.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_10.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_11.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_12.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_13.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_14.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_15.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_16.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_17.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_18.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_19.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_2.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_20.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_21.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_22.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_23.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_24.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_25.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_26.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_3.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_4.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_5.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_6.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_7.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_8.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_9.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_irreducible_graph_reset.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_empty_graph_reset.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_graph_id.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_graph_id_and_reset.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_graph_id_concurrent_rerecord.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_graph_reset.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_graph_reset_no_argument.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_if_implicit_taskgroup.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_exclusive.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_parallel.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_sequential.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_mutex_exclusive_serialized.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_mutex_irreducible_locks.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_mutex_runtime_locks.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_record_serial_replay_parallel.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_reduction_join.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_reset_during_replay.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_reset_no_leak.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_untied_parts.cpp
create mode 100644 openmp/runtime/test/tasking/omp_taskgraph_irreducible.cpp
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deadlock_1.cpp b/openmp/runtime/test/taskgraph/taskgraph_deadlock_1.cpp
new file mode 100644
index 00000000000000..65a37752be78dd
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deadlock_1.cpp
@@ -0,0 +1,55 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Non-lexically scoped tasks (without the 'replayable' clause) created while a
+// taskgraph is being recorded are not part of it, and must not deadlock the
+// recording.
+// The test should run to completion without crashing.
+
+#include <cstdio>
+
+int x = 0;
+int y = 10;
+int z = 20;
+
+// clang-format off
+__attribute__((noinline)) static void workOne() {
+#pragma omp task depend(inout : x)
+ {}
+}
+
+__attribute__((noinline)) static void workTwo() {
+#pragma omp task depend(inout : y, z)
+ {}
+}
+
+__attribute__((noinline)) static void workThree() {
+#pragma omp task depend(in : y, z) depend(out : x)
+ {}
+}
+// clang-format on
+
+int main() {
+#pragma omp parallel
+#pragma omp single
+ {
+ for (int i = 0; i < 10; ++i) {
+ // Would deadlock if the tasks are created out-of-line (not inlined) and
+ // the recorded graph is replayed on subsequent iterations.
+#pragma omp taskgraph
+ {
+ workOne();
+ workTwo();
+ workThree();
+ }
+ }
+ }
+
+ std::fprintf(stderr, "PASS taskgraph deadlock repro completed\n");
+ return 0;
+}
+
+// CHECK: PASS taskgraph deadlock repro completed
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_1.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_1.cpp
new file mode 100644
index 00000000000000..6c9241206bc60f
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_1.cpp
@@ -0,0 +1,53 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[3];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[2])
+ { }
+ #pragma omp task depend(out: deps[0], deps[1])
+ { }
+ #pragma omp task depend(inout: deps[0])
+ { }
+ #pragma omp task depend(inout: deps[1])
+ { }
+ #pragma omp task depend(inout: deps[2])
+ { }
+ #pragma omp task depend(in: deps[0], deps[1], deps[2])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_10.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_10.cpp
new file mode 100644
index 00000000000000..ff0a46b835706f
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_10.cpp
@@ -0,0 +1,50 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[5];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1], deps[4])
+ { }
+ #pragma omp task depend(in: deps[0], deps[1])
+ { }
+
+ #pragma omp task depend(out: deps[2], deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2], deps[3], deps[4])
+ { }
+ #pragma omp task depend(in: deps[2], deps[3])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x1c]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x7]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_11.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_11.cpp
new file mode 100644
index 00000000000000..308232707b9afe
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_11.cpp
@@ -0,0 +1,60 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[4];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(in: deps[0], deps[1])
+ { }
+
+ #pragma omp task depend(out: deps[2], deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+ { }
+ #pragma omp task depend(in: deps[2], deps[3])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: exclusive {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: exclusive {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_12.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_12.cpp
new file mode 100644
index 00000000000000..5a4fc884ff995f
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_12.cpp
@@ -0,0 +1,55 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[2];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(mutexinoutset: deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[1])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: exclusive {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: exclusive {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_13.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_13.cpp
new file mode 100644
index 00000000000000..6247283d0cfd67
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_13.cpp
@@ -0,0 +1,45 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[4];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(mutexinoutset: deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0xc]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x8]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x4]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x2]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x1]
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_14.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_14.cpp
new file mode 100644
index 00000000000000..ea7671f3345576
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_14.cpp
@@ -0,0 +1,48 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[4];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(mutexinoutset: deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1], deps[2])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1], deps[2], deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[1], deps[2], deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[3])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0xf]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0xe]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x7]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0xc]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x8]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x1]
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_15.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_15.cpp
new file mode 100644
index 00000000000000..f8659f9d12cb8a
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_15.cpp
@@ -0,0 +1,75 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[4];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[1], deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2], deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2], deps[1], deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[3], deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[3], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[3], deps[1], deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[3], deps[2])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[3], deps[2], deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[3], deps[2], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[3], deps[2], deps[1], deps[0])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0xf]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0xe]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0xd]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0xb]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x7]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0xc]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0xa]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x9]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x6]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x5]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x8]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x4]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x2]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x1]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_16.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_16.cpp
new file mode 100644
index 00000000000000..471e19fab88bf3
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_16.cpp
@@ -0,0 +1,55 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[8];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(inout: deps[0])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[4], deps[7])
+ { }
+ #pragma omp task depend(inout: deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[4], deps[7])
+ { }
+ #pragma omp task depend(inout: deps[2])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[5], deps[6])
+ { }
+ #pragma omp task depend(inout: deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[5], deps[6])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: exclusive {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: exclusive {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_17.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_17.cpp
new file mode 100644
index 00000000000000..fda89fd39a9a06
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_17.cpp
@@ -0,0 +1,68 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+ int deps[4];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[1])
+ {
+ fprintf(stderr, "task 0\n");
+ }
+ #pragma omp task depend(out: deps[2], deps[3])
+ {
+ fprintf(stderr, "task 1\n");
+ }
+ #pragma omp task depend(inout: deps[0])
+ {
+ fprintf(stderr, "task 2\n");
+ }
+ #pragma omp task depend(inout: deps[1])
+ {
+ fprintf(stderr, "task 3\n");
+ }
+ #pragma omp task depend(inout: deps[2])
+ {
+ fprintf(stderr, "task 4\n");
+ }
+ #pragma omp task depend(inout: deps[3])
+ {
+ fprintf(stderr, "task 5\n");
+ }
+ #pragma omp task depend(in: deps[0], deps[1], deps[2], deps[3])
+ {
+ fprintf(stderr, "task 6\n");
+ }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 2
+// CHECK-DAG: task 3
+// CHECK-DAG: task 1
+// CHECK-DAG: task 4
+// CHECK-DAG: task 5
+// CHECK: task 6
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 2
+// CHECK-DAG: task 3
+// CHECK-DAG: task 1
+// CHECK-DAG: task 4
+// CHECK-DAG: task 5
+// CHECK: task 6
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_18.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_18.cpp
new file mode 100644
index 00000000000000..a04e405de6c700
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_18.cpp
@@ -0,0 +1,46 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[2];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[1])
+ { }
+ #pragma omp taskloop num_tasks(strict: 2)
+ {
+ for (int j = 0; j < 20; j++) { }
+ }
+ #pragma omp task depend(in: deps[0], deps[1])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: wait: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_19.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_19.cpp
new file mode 100644
index 00000000000000..b18f8c844e8e13
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_19.cpp
@@ -0,0 +1,51 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+ int deps[3];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 10; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0])
+ {
+ fprintf(stderr, "task 0\n");
+ }
+ #pragma omp task depend(out: deps[1])
+ {
+ fprintf(stderr, "task 1\n");
+ }
+ #pragma omp task depend(out: deps[2])
+ {
+ fprintf(stderr, "task 2\n");
+ }
+ #pragma omp taskwait depend(in: deps[0], deps[1], deps[2])
+ #pragma omp task depend(in: deps[0], deps[1], deps[2])
+ {
+ fprintf(stderr, "task 3\n");
+ }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 1
+// CHECK-DAG: task 2
+// CHECK: task 3
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 1
+// CHECK-DAG: task 2
+// CHECK: task 3
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_2.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_2.cpp
new file mode 100644
index 00000000000000..7b897bc92bb621
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_2.cpp
@@ -0,0 +1,58 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[4];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[1])
+ { }
+ #pragma omp task depend(out: deps[2], deps[3])
+ { }
+ #pragma omp task depend(inout: deps[0])
+ { }
+ #pragma omp task depend(inout: deps[1])
+ { }
+ #pragma omp task depend(inout: deps[2])
+ { }
+ #pragma omp task depend(inout: deps[3])
+ { }
+ #pragma omp task depend(in: deps[0], deps[1], deps[2], deps[3])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_20.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_20.cpp
new file mode 100644
index 00000000000000..549b6d6138a528
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_20.cpp
@@ -0,0 +1,51 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+ int deps[3];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 10; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0])
+ {
+ fprintf(stderr, "task 0\n");
+ }
+ #pragma omp task depend(out: deps[1])
+ {
+ fprintf(stderr, "task 1\n");
+ }
+ #pragma omp task depend(out: deps[2])
+ {
+ fprintf(stderr, "task 2\n");
+ }
+ #pragma omp taskwait depend(inoutset: deps[0], deps[1])
+ #pragma omp task depend(in: deps[0], deps[1], deps[2])
+ {
+ fprintf(stderr, "task 3\n");
+ }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 1
+// CHECK-DAG: task 2
+// CHECK: task 3
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 1
+// CHECK-DAG: task 2
+// CHECK: task 3
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_21.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_21.cpp
new file mode 100644
index 00000000000000..0b86e677a13ebd
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_21.cpp
@@ -0,0 +1,52 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+ int arr[100];
+
+ int res = 0;
+ for (int i = 0; i < 100; i++) {
+ arr[i] = i;
+ res += i;
+ }
+ printf("base result: %d\n", res);
+
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 10; i++)
+ {
+ int res = 0;
+ #pragma omp taskgraph
+ {
+ #pragma omp taskloop reduction(+: res) num_tasks(10)
+ {
+ for (int j = 0; j < 100; j++) {
+ res += arr[j];
+ }
+ }
+ }
+ printf("reduction result: %d\n", res);
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: base result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_22.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_22.cpp
new file mode 100644
index 00000000000000..66b1ec8b8093cf
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_22.cpp
@@ -0,0 +1,70 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+void foo() {
+#pragma omp task replayable(1)
+ {
+ fprintf(stderr, "task outside lexical taskgraph\n");
+ }
+}
+
+int main()
+{
+ int arr[100];
+
+ int res = 0;
+ for (int i = 0; i < 100; i++) {
+ arr[i] = i;
+ res += i;
+ }
+ fprintf(stderr, "base result: %d\n", res);
+
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 10; i++)
+ {
+ int res = 0;
+ #pragma omp taskgraph
+ {
+ #pragma omp taskloop reduction(+: res) num_tasks(10)
+ {
+ for (int j = 0; j < 100; j++) {
+ res += arr[j];
+ }
+ }
+ foo();
+ }
+ fprintf(stderr, "reduction result: %d\n", res);
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: base result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_23.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_23.cpp
new file mode 100644
index 00000000000000..d5ed9cba60cdac
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_23.cpp
@@ -0,0 +1,103 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+void foo() {
+ fprintf(stderr, "called function foo\n");
+#pragma omp taskloop replayable num_tasks(4)
+ {
+ for (int i = 0; i < 4; i++)
+ fprintf(stderr, "taskloop iter %d outside lexical taskgraph\n", i);
+ }
+}
+
+int main()
+{
+ int arr[100];
+
+ int res = 0;
+ for (int i = 0; i < 100; i++) {
+ arr[i] = i;
+ res += i;
+ }
+ fprintf(stderr, "base result: %d\n", res);
+
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 10; i++)
+ {
+ int res = 0;
+ #pragma omp taskgraph
+ {
+ #pragma omp taskloop reduction(+: res) num_tasks(10)
+ {
+ for (int j = 0; j < 100; j++) {
+ res += arr[j];
+ }
+ }
+ foo();
+ }
+ fprintf(stderr, "reduction result: %d\n", res);
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: base result: 4950
+// CHECK-NEXT: called function foo
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_24.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_24.cpp
new file mode 100644
index 00000000000000..cb6878c82fe2f2
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_24.cpp
@@ -0,0 +1,80 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cassert>
+
+int global_dep;
+
+void foo() {
+#pragma omp taskwait replayable(1) depend(in: global_dep)
+}
+
+int main()
+{
+ int arr[100];
+
+ int res = 0;
+ for (int i = 0; i < 100; i++) {
+ arr[i] = i;
+ res += i;
+ }
+
+ assert(res == 4950);
+
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 10; i++)
+ {
+ int res = 0;
+ #pragma omp taskgraph
+ {
+ #pragma omp taskloop reduction(+: res) num_tasks(10)
+ {
+ for (int j = 0; j < 100; j++) {
+ res += arr[j];
+ }
+ }
+ #pragma omp task depend(out: global_dep)
+ { }
+ foo();
+ }
+ assert(res == 4950);
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: wait: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: wait: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_25.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_25.cpp
new file mode 100644
index 00000000000000..893f51bc1e0fb7
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_25.cpp
@@ -0,0 +1,89 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int global_dep;
+
+void foo() {
+ fprintf(stderr, "called function foo\n");
+#pragma omp task replayable(1) depend(in: global_dep)
+ {
+ fprintf(stderr, "out-of-line task created from within taskloop\n");
+ }
+}
+
+int main()
+{
+ int arr[100];
+
+ int res = 0;
+ for (int i = 0; i < 4; i++) {
+ arr[i] = i;
+ res += i;
+ }
+ fprintf(stderr, "base result: %d\n", res);
+
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 4; i++)
+ {
+ int res = 0;
+ #pragma omp taskgraph
+ {
+ #pragma omp taskloop reduction(+: res) num_tasks(4)
+ {
+ for (int j = 0; j < 4; j++) {
+ res += arr[j];
+ foo();
+ }
+ }
+ }
+ fprintf(stderr, "reduction result: %d\n", res);
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: base result: 6
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: reduction result: 6
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: reduction result: 6
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: reduction result: 6
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: reduction result: 6
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_26.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_26.cpp
new file mode 100644
index 00000000000000..d98b4b60c485e4
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_26.cpp
@@ -0,0 +1,61 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+ int arr[100];
+ int arr2[100];
+
+ int res = 0, res2 = 0;
+ for (int i = 0; i < 10; i++) {
+ arr[i] = i;
+ arr2[i] = 3 + i * 2;
+ res += i;
+ res2 += 3 + i * 2;
+ }
+ fprintf(stderr, "base results: %d, %d\n", res, res2);
+
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 10; i++)
+ {
+ int res = 0, res2 = 0;
+ #pragma omp taskgraph
+ {
+ #pragma omp taskloop reduction(+: res) num_tasks(10)
+ {
+ for (int j = 0; j < 10; j++) {
+ res += arr[j];
+ }
+ }
+ #pragma omp taskloop reduction(+: res2) num_tasks(10)
+ {
+ for (int j = 0; j < 10; j++) {
+ res2 += arr2[j];
+ }
+ }
+ }
+ fprintf(stderr, "reduction results: %d, %d\n", res, res2);
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: base results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp
new file mode 100644
index 00000000000000..fd2323c21fe5f4
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp
@@ -0,0 +1,63 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+ int arr[100];
+ int arr2[100];
+
+ int res = 0, res2 = 0;
+ for (int i = 0; i < 10; i++) {
+ arr[i] = i;
+ res += i;
+ }
+ for (int i = 0; i < 10; i++) {
+ arr2[i] = 3 + i * 2;
+ res2 += res * (3 + i * 2);
+ }
+ fprintf(stderr, "base results: %d, %d\n", res, res2);
+
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 10; i++)
+ {
+ int res = 0, res2 = 0;
+ #pragma omp taskgraph
+ {
+ #pragma omp taskloop reduction(+: res) num_tasks(10)
+ {
+ for (int j = 0; j < 10; j++) {
+ res += arr[j];
+ }
+ }
+ #pragma omp taskloop reduction(+: res2) num_tasks(10)
+ {
+ for (int j = 0; j < 10; j++) {
+ res2 += res * arr2[j];
+ }
+ }
+ }
+ fprintf(stderr, "reduction results: %d, %d\n", res, res2);
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: base results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_3.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_3.cpp
new file mode 100644
index 00000000000000..196a705e516f0d
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_3.cpp
@@ -0,0 +1,62 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[6];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[1])
+ { }
+ #pragma omp task depend(out: deps[2], deps[3])
+ { }
+ #pragma omp task depend(inout: deps[0])
+ { }
+ #pragma omp task depend(inout: deps[1])
+ { }
+ #pragma omp task depend(inout: deps[2])
+ { }
+ #pragma omp task depend(inout: deps[3])
+ { }
+ #pragma omp task depend(in: deps[0], deps[1], deps[2], deps[3])
+ { }
+ #pragma omp task depend(in: deps[1], deps[2]) depend(out: deps[5])
+ { }
+ #pragma omp task depend(in: deps[5])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// This dependence graph is irreducible (it has no series-parallel
+// decomposition), so the residual tangle is carved into a single
+// TASKGRAPH_REGION_IRREDUCIBLE container whose children carry explicit intra
+// edges. The two tasks that form a clean producer/consumer chain on deps[5]
+// (the depend(out: deps[5]) and depend(in: deps[5]) tasks) still reduce to a
+// nested sequential.
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: irreducible {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_4.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_4.cpp
new file mode 100644
index 00000000000000..d86633ae51e259
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_4.cpp
@@ -0,0 +1,59 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[4];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[1])
+ { }
+ #pragma omp task depend(out: deps[2], deps[3])
+ { }
+ #pragma omp task depend(inout: deps[0])
+ { }
+ #pragma omp task depend(inout: deps[1])
+ { }
+ #pragma omp task depend(inout: deps[2])
+ { }
+ #pragma omp task depend(inout: deps[3])
+ { }
+ #pragma omp task depend(in: deps[0], deps[2], deps[3])
+ { }
+ #pragma omp task depend(in: deps[0], deps[1])
+ { }
+ #pragma omp task depend(in: deps[3])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// This dependence graph is irreducible (it has no series-parallel
+// decomposition), so the whole residual tangle is carved into a single
+// TASKGRAPH_REGION_IRREDUCIBLE container whose children (the nine task arms)
+// carry explicit intra edges rather than being nested into parallel/sequential
+// regions.
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: irreducible {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_5.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_5.cpp
new file mode 100644
index 00000000000000..18bc54a164b4c8
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_5.cpp
@@ -0,0 +1,63 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[4];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[2])
+ { }
+ #pragma omp task depend(out: deps[1], deps[3])
+ { }
+ #pragma omp task depend(inoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(inoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(inoutset: deps[2], deps[3])
+ { }
+ #pragma omp task depend(inoutset: deps[2], deps[3])
+ { }
+ #pragma omp task depend(in: deps[0], deps[1])
+ { }
+ #pragma omp task depend(in: deps[2], deps[3])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_6.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_6.cpp
new file mode 100644
index 00000000000000..c7190eae0d4890
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_6.cpp
@@ -0,0 +1,59 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[4];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[1], deps[3])
+ { }
+ #pragma omp task depend(out: deps[0], deps[2])
+ { }
+ #pragma omp task depend(inoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(inoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(inoutset: deps[2], deps[3])
+ { }
+ #pragma omp task depend(inoutset: deps[2], deps[3])
+ { }
+ #pragma omp task depend(in: deps[0], deps[2])
+ { }
+ #pragma omp task depend(in: deps[0], deps[2])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_7.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_7.cpp
new file mode 100644
index 00000000000000..38f60b16314d6c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_7.cpp
@@ -0,0 +1,59 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[4];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[1])
+ { }
+ #pragma omp task depend(out: deps[2], deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+ { }
+ #pragma omp task depend(in: deps[0], deps[1])
+ { }
+ #pragma omp task depend(in: deps[2], deps[3])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: exclusive {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: exclusive {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_8.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_8.cpp
new file mode 100644
index 00000000000000..c723212290a162
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_8.cpp
@@ -0,0 +1,39 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[2];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(in: deps[0], deps[1])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// FIXME: This isn't perfect -- we don't really need to keep the mutexes in
+// this case.
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_9.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_9.cpp
new file mode 100644
index 00000000000000..1c9e99337452b9
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_9.cpp
@@ -0,0 +1,47 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+ int deps[3];
+ #pragma omp parallel
+ {
+ #pragma omp single
+ {
+ for (int i = 0; i < 2; i++)
+ {
+ #pragma omp taskgraph
+ {
+ #pragma omp task depend(out: deps[0], deps[1])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(in: deps[1]) depend(out: deps[2])
+ { }
+ #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+ { }
+ #pragma omp task depend(in: deps[0], deps[1], deps[2])
+ { }
+ }
+ }
+ }
+ }
+ return 0;
+}
+
+// FIXME: This isn't perfect -- we don't really need to keep the mutexes in
+// this case.
+
+// CHECK: Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: sequential {
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT: node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_irreducible_graph_reset.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_irreducible_graph_reset.cpp
new file mode 100644
index 00000000000000..0e94a1644cd994
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_irreducible_graph_reset.cpp
@@ -0,0 +1,50 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <atomic>
+#include <cstdio>
+
+// This builds a non-series-parallel dependency shape:
+// A->B, A->C, B->D, C->D, C->E.
+
+int main() {
+ int deps[4] = {0, 0, 0, 0};
+
+ for (int iter = 0; iter < 1000; ++iter) {
+ std::atomic<int> sum{0};
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+#pragma omp taskgraph graph_id(123) graph_reset(1)
+ {
+#pragma omp task depend(out : deps[0], deps[1])
+ { sum.fetch_add(1, std::memory_order_relaxed); } // A
+#pragma omp task depend(inout : deps[0])
+ { sum.fetch_add(4, std::memory_order_relaxed); } // B
+#pragma omp task depend(inout : deps[1])
+ { sum.fetch_add(8, std::memory_order_relaxed); } // C
+#pragma omp task depend(in : deps[0], deps[1], deps[2], deps[3])
+ { sum.fetch_add(64, std::memory_order_relaxed); } // D
+#pragma omp task depend(in : deps[1], deps[2])
+ { sum.fetch_add(128, std::memory_order_relaxed); } // E
+ }
+ }
+ }
+
+ const int actual = sum.load(std::memory_order_relaxed);
+ if (actual != 205) {
+ std::fprintf(stderr, "FAIL iter=%d expected=205 actual=%d\n", iter,
+ actual);
+ return 1;
+ }
+ }
+
+ std::fprintf(stderr, "PASS\n");
+ return 0;
+}
+
+// CHECK: PASS
diff --git a/openmp/runtime/test/taskgraph/taskgraph_empty_graph_reset.cpp b/openmp/runtime/test/taskgraph/taskgraph_empty_graph_reset.cpp
new file mode 100644
index 00000000000000..5a51963bc06ffa
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_empty_graph_reset.cpp
@@ -0,0 +1,58 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A recording that ends up with no task nodes is left with a null region tree
+// by __kmp_build_taskgraph, so tearing such a record down (which is what
+// graph_reset does before re-recording) must tolerate that.
+
+#include <atomic>
+#include <cstdio>
+
+int main() {
+ constexpr int NumIters = 4;
+
+ std::atomic<int> total{0};
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ // An empty body: nothing is recorded, so the record has no regions.
+ for (int iter = 0; iter < NumIters; ++iter) {
+#pragma omp taskgraph graph_id(1) graph_reset(1)
+ {
+ }
+ }
+
+ // Same, but alternating empty and non-empty bodies, so a re-record has
+ // to cope with the previous record being empty and vice versa.
+ for (int iter = 0; iter < NumIters; ++iter) {
+#pragma omp taskgraph graph_id(2) graph_reset(1)
+ {
+ if (iter & 1) {
+ for (int i = 0; i < 4; ++i) {
+#pragma omp task
+ total.fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ const int Expected = (NumIters / 2) * 4;
+ const int Actual = total.load(std::memory_order_relaxed);
+
+ if (Actual != Expected) {
+ std::fprintf(stderr, "FAIL empty graph_reset total=%d expected=%d\n",
+ Actual, Expected);
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS empty graph_reset total=%d\n", Actual);
+ return 0;
+}
+
+// CHECK: PASS empty graph_reset total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_graph_id.cpp b/openmp/runtime/test/taskgraph/taskgraph_graph_id.cpp
new file mode 100644
index 00000000000000..f536c2a03c9b1c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_graph_id.cpp
@@ -0,0 +1,56 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <atomic>
+#include <cstdio>
+
+int main() {
+ constexpr int NumIters = 12;
+ constexpr int WorkA = 16;
+ constexpr int WorkB = 24;
+
+ std::atomic<int> total{0};
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ for (int iter = 0; iter < NumIters; ++iter) {
+ const int gid = iter & 1;
+
+#pragma omp taskgraph graph_id(gid)
+ {
+ if (gid == 0) {
+ for (int i = 0; i < WorkA; ++i) {
+#pragma omp task
+ total.fetch_add(1, std::memory_order_relaxed);
+ }
+ } else {
+ for (int i = 0; i < WorkB; ++i) {
+#pragma omp task
+ total.fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ const int NumEven = (NumIters + 1) / 2;
+ const int NumOdd = NumIters / 2;
+ const int Expected = NumEven * WorkA + NumOdd * WorkB;
+ const int Actual = total.load(std::memory_order_relaxed);
+
+ if (Actual != Expected) {
+ std::fprintf(stderr, "FAIL graph_id total=%d expected=%d\n", Actual,
+ Expected);
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS graph_id total=%d\n", Actual);
+ return 0;
+}
+
+// CHECK: PASS graph_id total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_graph_id_and_reset.cpp b/openmp/runtime/test/taskgraph/taskgraph_graph_id_and_reset.cpp
new file mode 100644
index 00000000000000..cadc527102ee72
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_graph_id_and_reset.cpp
@@ -0,0 +1,57 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <atomic>
+#include <cstdio>
+
+// Exercise graph_id and graph_reset together on a single taskgraph construct.
+//
+// graph_id is per-taskgraph: the runtime keeps one record per (construct,
+// graph_id) pair. Here we feed the same construct two different graph_ids
+// alternately (0, 1, 0, 1, ...) so two records coexist for the same
+// directive. graph_reset is then driven by a separate condition: every
+// fourth visit, we ask the runtime to re-record the graph_id we are about
+// to use, forcing it through the expiry-and-re-record path while the other
+// graph_id's record stays intact and continues to be replayed.
+
+int main() {
+ constexpr int NumIters = 40;
+ constexpr int TasksPerVisit = 6;
+
+ std::atomic<int> total{0};
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ for (int iter = 0; iter < NumIters; ++iter) {
+ const int gid = iter & 1; // alternate two records
+ const bool reset = (iter % 4) == 3; // periodically force re-record
+
+#pragma omp taskgraph graph_id(gid) graph_reset(reset)
+ {
+ for (int i = 0; i < TasksPerVisit; ++i) {
+#pragma omp task
+ total.fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+ }
+ }
+ }
+
+ const int Expected = NumIters * TasksPerVisit;
+ const int Actual = total.load(std::memory_order_relaxed);
+
+ if (Actual != Expected) {
+ std::fprintf(stderr, "FAIL graph_id+reset total=%d expected=%d\n", Actual,
+ Expected);
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS graph_id+reset total=%d\n", Actual);
+ return 0;
+}
+
+// CHECK: PASS graph_id+reset total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_graph_id_concurrent_rerecord.cpp b/openmp/runtime/test/taskgraph/taskgraph_graph_id_concurrent_rerecord.cpp
new file mode 100644
index 00000000000000..385e1f7ed795d0
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_graph_id_concurrent_rerecord.cpp
@@ -0,0 +1,50 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_DYNAMIC=FALSE %not --crash %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <atomic>
+#include <cstdio>
+#include <omp.h>
+
+static std::atomic<int> first_thread_inside{0};
+static std::atomic<int> release_first_thread{0};
+
+// Deterministically force two threads to hit the same lexical taskgraph
+// construct with the same graph_id while the first thread is still recording
+// it. That guarantees the second thread sees an existing record in
+// KMP_TDG_RECORDING state and trips the runtime sanity check.
+static void enter_same_taskgraph(int tid) {
+#pragma omp taskgraph graph_id(17) graph_reset(1)
+ {
+ if (tid == 0) {
+ first_thread_inside.store(1, std::memory_order_release);
+ while (release_first_thread.load(std::memory_order_acquire) == 0) {
+ }
+ }
+ }
+}
+
+int main() {
+#pragma omp parallel num_threads(2)
+ {
+ const int tid = omp_get_thread_num();
+
+ if (tid == 1) {
+ while (first_thread_inside.load(std::memory_order_acquire) == 0) {
+ }
+ }
+
+ enter_same_taskgraph(tid);
+
+ if (tid == 1)
+ release_first_thread.store(1, std::memory_order_release);
+ }
+
+ std::fprintf(stderr, "UNEXPECTED SUCCESS\n");
+ return 0;
+}
+
+// CHECK: OMP: Error #{{[0-9]+}}: {{.*}}multiple threads attempting to re-record
+// CHECK-SAME: taskgraph concurrently for graph_id 17
+// CHECK-NOT: UNEXPECTED SUCCESS
diff --git a/openmp/runtime/test/taskgraph/taskgraph_graph_reset.cpp b/openmp/runtime/test/taskgraph/taskgraph_graph_reset.cpp
new file mode 100644
index 00000000000000..d41592f0ac243e
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_graph_reset.cpp
@@ -0,0 +1,57 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <atomic>
+#include <cstdio>
+
+int main() {
+ constexpr int NumIters = 10;
+ constexpr int GraphId = 7;
+
+ std::atomic<int> total{0};
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ for (int iter = 0; iter < NumIters; ++iter) {
+ const bool odd = (iter & 1) != 0;
+
+ // Alternate the taskgraph shape every iteration. graph_reset(1)
+ // requires the runtime to re-record rather than replay stale shape.
+#pragma omp taskgraph graph_id(GraphId) graph_reset(1)
+ {
+ if (odd) {
+ for (int i = 0; i < 40; ++i) {
+#pragma omp task
+ total.fetch_add(2, std::memory_order_relaxed);
+ }
+ } else {
+ for (int i = 0; i < 10; ++i) {
+#pragma omp task
+ total.fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ const int NumOdd = NumIters / 2;
+ const int NumEven = NumIters - NumOdd;
+ const int Expected = NumOdd * 40 * 2 + NumEven * 10;
+ const int Actual = total.load(std::memory_order_relaxed);
+
+ if (Actual != Expected) {
+ std::fprintf(stderr, "FAIL graph_reset total=%d expected=%d\n", Actual,
+ Expected);
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS graph_reset total=%d\n", Actual);
+ return 0;
+}
+
+// CHECK: PASS graph_reset total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_graph_reset_no_argument.cpp b/openmp/runtime/test/taskgraph/taskgraph_graph_reset_no_argument.cpp
new file mode 100644
index 00000000000000..1f698a6ae06138
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_graph_reset_no_argument.cpp
@@ -0,0 +1,64 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// The graph_reset argument has the optional property, so a bare 'graph_reset'
+// means reset unconditionally -- the same as graph_reset(1), and in particular
+// not the same as no graph_reset clause at all. This is the runtime half of
+// clang/test/OpenMP/taskgraph_clauses_codegen.cpp: it pins the behaviour rather
+// than the constant that is passed.
+
+#include <atomic>
+#include <cstdio>
+
+int main() {
+ constexpr int NumIters = 10;
+ constexpr int GraphId = 11;
+
+ std::atomic<int> total{0};
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ for (int iter = 0; iter < NumIters; ++iter) {
+ const bool odd = (iter & 1) != 0;
+
+ // Alternate the taskgraph shape every iteration. Without a reset the
+ // first-recorded shape would be replayed every time, so the total only
+ // comes out right if the bare clause really does re-record.
+#pragma omp taskgraph graph_id(GraphId) graph_reset
+ {
+ if (odd) {
+ for (int i = 0; i < 40; ++i) {
+#pragma omp task
+ total.fetch_add(2, std::memory_order_relaxed);
+ }
+ } else {
+ for (int i = 0; i < 10; ++i) {
+#pragma omp task
+ total.fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ const int NumOdd = NumIters / 2;
+ const int NumEven = NumIters - NumOdd;
+ const int Expected = NumOdd * 40 * 2 + NumEven * 10;
+ const int Actual = total.load(std::memory_order_relaxed);
+
+ if (Actual != Expected) {
+ std::fprintf(stderr, "FAIL graph_reset total=%d expected=%d\n", Actual,
+ Expected);
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS graph_reset total=%d\n", Actual);
+ return 0;
+}
+
+// CHECK: PASS graph_reset total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_if_implicit_taskgroup.cpp b/openmp/runtime/test/taskgraph/taskgraph_if_implicit_taskgroup.cpp
new file mode 100644
index 00000000000000..2856e2c8ecc00b
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_if_implicit_taskgroup.cpp
@@ -0,0 +1,90 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A taskgraph region is a taskgroup region whether or not it records: the 'if'
+// clause governs recording, and 'nogroup' is the only way to drop the implicit
+// taskgroup. So the tasks a taskgraph generates must have completed by the end
+// of the region for *both* values of the condition.
+//
+// The recording path gets its taskgroup from __kmpc_taskgraph. The if(false)
+// path bypasses that call entirely, so it needs a taskgroup of its own; without
+// one the generated tasks would still be running after the region had ended
+// -- silent under-synchronisation rather than a crash, hence this test.
+//
+// Each task sleeps before reporting, so an un-waited-for region is caught
+// rather than merely raced on: with the taskgroup missing, the check below
+// reliably sees a count of zero.
+
+#include <atomic>
+#include <cstdio>
+#include <unistd.h>
+
+static constexpr int NumIters = 4;
+static constexpr int NumTasks = 8;
+
+static std::atomic<int> done;
+
+// Returns the number of tasks that had not finished by the end of the region.
+static int missing_at_end_of_region(bool record) {
+ int outstanding = 0;
+
+ for (int iter = 0; iter < NumIters; ++iter) {
+ done.store(0, std::memory_order_relaxed);
+
+ if (record) {
+#pragma omp taskgraph graph_id(1) if (true)
+ {
+ for (int i = 0; i < NumTasks; ++i) {
+#pragma omp task
+ {
+ usleep(1000);
+ done.fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+ }
+ } else {
+#pragma omp taskgraph graph_id(2) if (false)
+ {
+ for (int i = 0; i < NumTasks; ++i) {
+#pragma omp task
+ {
+ usleep(1000);
+ done.fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+ }
+ }
+
+ outstanding += NumTasks - done.load(std::memory_order_relaxed);
+ }
+
+ return outstanding;
+}
+
+int main() {
+ int recorded = -1, not_recorded = -1;
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ recorded = missing_at_end_of_region(/*record=*/true);
+ not_recorded = missing_at_end_of_region(/*record=*/false);
+ }
+ }
+
+ if (recorded != 0 || not_recorded != 0) {
+ std::fprintf(stderr,
+ "FAIL taskgraph if: outstanding tasks if(true)=%d "
+ "if(false)=%d, expected 0 and 0\n",
+ recorded, not_recorded);
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS taskgraph if implicit taskgroup\n");
+ return 0;
+}
+
+// CHECK: PASS taskgraph if implicit taskgroup
diff --git a/openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_exclusive.cpp b/openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_exclusive.cpp
new file mode 100644
index 00000000000000..b3ee58f884ebee
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_exclusive.cpp
@@ -0,0 +1,82 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// The same shape as taskgraph_irreducible_nested_sequential.cpp reached through
+// the other container type that carries no exec descriptor of its own.
+//
+// T1 T2
+// / \ / \
+// T4 \ / T5 excl = { X | Y } (mutexinoutset)
+// | [ excl ] |
+// | / \ |
+// T7 -' '-- T8
+//
+// X and Y have identical predecessor and successor sets, so the twin-merge pass
+// folds them into one region before the knot is carved; because they carry a
+// mutexinoutset, the merge produces an EXCLUSIVE rather than a PARALLEL. Like
+// SEQUENTIAL, EXCLUSIVE is expanded by recursing into its children and is never
+// given a descriptor, so an edge naming it has to resolve to the exit descrs of
+// its last child.
+//
+// Being the only shape that puts a mutex set inside a knot, this also covers
+// __kmp_taskgraph_gather_mutex_sets descending into an IRREDUCIBLE: without
+// that, the EXCLUSIVE keeps a null mutexset and
+// __kmp_taskgraph_find_exclusive_regions dereferences it.
+
+#include <cstdio>
+
+#define ITERS 200
+
+static volatile int a, b, x, y, d, e;
+static volatile int m;
+static int errors;
+
+int main() {
+ for (int iter = 0; iter < ITERS; ++iter) {
+ a = b = x = y = d = e = 0;
+#pragma omp parallel
+#pragma omp single
+ {
+#pragma omp taskgraph
+ {
+ // clang-format off
+#pragma omp task depend(out : a)
+ { a = 1; }
+#pragma omp task depend(out : b)
+ { b = 2; }
+#pragma omp task depend(in : a, b) depend(mutexinoutset : m) depend(out : x)
+ {
+ m = m + 1;
+ x = a + b;
+ }
+#pragma omp task depend(in : a, b) depend(mutexinoutset : m) depend(out : y)
+ {
+ m = m + 1;
+ y = a + b + 1;
+ }
+#pragma omp task depend(in : a) depend(out : d)
+ { d = a + 10; }
+#pragma omp task depend(in : b) depend(out : e)
+ { e = b + 20; }
+ // clang-format on
+#pragma omp task depend(in : x, y, d)
+ {
+ if (x != 3 || y != 4 || d != 11)
+ ++errors;
+ }
+#pragma omp task depend(in : x, y, e)
+ {
+ if (x != 3 || y != 4 || e != 22)
+ ++errors;
+ }
+ }
+ }
+ }
+
+ std::printf("irreducible nested exclusive errors: %d\n", errors);
+ return errors != 0;
+}
+
+// CHECK: irreducible nested exclusive errors: 0
diff --git a/openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_parallel.cpp b/openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_parallel.cpp
new file mode 100644
index 00000000000000..63d650796c9ff1
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_parallel.cpp
@@ -0,0 +1,87 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A knot (irreducible region) one of whose members is a collapsed PARALLEL
+// container that other members of the same knot depend on.
+//
+// T1 T2
+// / \ / \
+// T4 \ / T5 par = { X | Y }
+// | [ par ] |
+// | / \ |
+// T7 -' '-- T8
+//
+// X and Y have identical predecessor and successor sets, so the twin-merge pass
+// folds them into one PARALLEL region before the knot is carved; X and Y then
+// feed T7 and T8, so the PARALLEL appears in their 'predecessors' lists.
+//
+// A PARALLEL does get an exec descriptor of its own, but it is the container's
+// *gather* (entry) descriptor, which fires when the container starts. An edge
+// naming the PARALLEL must therefore resolve to its exit set -- both X and Y,
+// since they run concurrently -- and not to that entry descriptor, or T7 and T8
+// are released too early. X and Y spin briefly to make an early release
+// observable rather than merely possible.
+
+#include <cstdio>
+
+#define ITERS 200
+#define SPIN 200000
+
+static volatile int a, b, x, y, d, e;
+static volatile int sink;
+static int errors;
+
+static void spin() {
+ for (int i = 0; i < SPIN; ++i)
+ sink = sink + 1;
+}
+
+int main() {
+ for (int iter = 0; iter < ITERS; ++iter) {
+ a = b = x = y = d = e = 0;
+#pragma omp parallel
+#pragma omp single
+ {
+#pragma omp taskgraph
+ {
+ // clang-format off
+#pragma omp task depend(out : a)
+ { a = 1; }
+#pragma omp task depend(out : b)
+ { b = 2; }
+#pragma omp task depend(in : a, b) depend(out : x)
+ {
+ spin();
+ x = a + b;
+ }
+#pragma omp task depend(in : a, b) depend(out : y)
+ {
+ spin();
+ y = a + b + 1;
+ }
+#pragma omp task depend(in : a) depend(out : d)
+ { d = a + 10; }
+#pragma omp task depend(in : b) depend(out : e)
+ { e = b + 20; }
+ // clang-format on
+#pragma omp task depend(in : x, y, d)
+ {
+ if (x != 3 || y != 4 || d != 11)
+ ++errors;
+ }
+#pragma omp task depend(in : x, y, e)
+ {
+ if (x != 3 || y != 4 || e != 22)
+ ++errors;
+ }
+ }
+ }
+ }
+
+ std::printf("irreducible nested parallel errors: %d\n", errors);
+ return errors != 0;
+}
+
+// CHECK: irreducible nested parallel errors: 0
diff --git a/openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_sequential.cpp b/openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_sequential.cpp
new file mode 100644
index 00000000000000..49194d9522541e
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_irreducible_nested_sequential.cpp
@@ -0,0 +1,75 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A knot (irreducible region) one of whose members is a collapsed SEQUENTIAL
+// container that other members of the same knot depend on.
+//
+// T1 T2
+// / \ / \
+// T4 \ / T5 seq = (T3 ; T6)
+// | [ seq ] |
+// | / \ |
+// T7 -' '-- T8
+//
+// T3 -> T6 is a clean 1:1 producer/consumer chain, so the series collapse folds
+// it into a SEQUENTIAL region before the residual knot is carved; T6 then feeds
+// T7 and T8, so the SEQUENTIAL appears in their 'predecessors' lists.
+//
+// A SEQUENTIAL (like an EXCLUSIVE) gets no exec descriptor of its own, so an
+// edge naming it has to resolve to the exit descrs of its last child. Note the
+// carved members are in reverse *pre*order, not topological order
+// (__kmp_taskgraph_region_dfs numbers on entry), so the SEQUENTIAL is reached
+// from T7 before it would have been built in member order.
+
+#include <cstdio>
+
+#define ITERS 200
+
+static volatile int a, b, c, d, e, f;
+static int errors;
+
+int main() {
+ for (int iter = 0; iter < ITERS; ++iter) {
+ a = b = c = d = e = f = 0;
+#pragma omp parallel
+#pragma omp single
+ {
+#pragma omp taskgraph
+ {
+ // clang-format off
+#pragma omp task depend(out : a)
+ { a = 1; }
+#pragma omp task depend(out : b)
+ { b = 2; }
+#pragma omp task depend(in : a, b) depend(out : c)
+ { c = a + b; }
+#pragma omp task depend(in : a) depend(out : d)
+ { d = a + 10; }
+#pragma omp task depend(in : b) depend(out : e)
+ { e = b + 20; }
+#pragma omp task depend(in : c) depend(out : f)
+ { f = c + 100; }
+ // clang-format on
+#pragma omp task depend(in : d, f)
+ {
+ if (d != 11 || f != 103)
+ ++errors;
+ }
+#pragma omp task depend(in : e, f)
+ {
+ if (e != 22 || f != 103)
+ ++errors;
+ }
+ }
+ }
+ if (c != 3 || f != 103)
+ ++errors;
+ }
+
+ std::printf("irreducible nested sequential errors: %d\n", errors);
+ return errors != 0;
+}
+
+// CHECK: irreducible nested sequential errors: 0
diff --git a/openmp/runtime/test/taskgraph/taskgraph_mutex_exclusive_serialized.cpp b/openmp/runtime/test/taskgraph/taskgraph_mutex_exclusive_serialized.cpp
new file mode 100644
index 00000000000000..dc2f621c2cac9b
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_mutex_exclusive_serialized.cpp
@@ -0,0 +1,83 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+// RUN: env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck --check-prefix=SHAPE --implicit-check-not='sets:' %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Mutual exclusion for a mutexinoutset that the region builder can express
+// structurally. The two mutex tasks have identical predecessor and successor
+// sets, so the twin-merge pass folds them into one EXCLUSIVE region, whose
+// children are expanded in sequence:
+//
+// T1 (out: d)
+// |
+// [ exclusive ] both hold { d } as mutexinoutset
+// / \
+// TA TB
+// |
+// T4 (in: d)
+//
+// __kmp_taskgraph_strip_mutex_sets then drops the set (in_exclusive), so replay
+// is ordered purely by graph shape and no replay-time lock is involved. This
+// is the control for taskgraph_mutex_runtime_locks.cpp, which covers the shape
+// the builder cannot express structurally and which therefore does need locks.
+//
+// The two paths give the same observable answer, so the SHAPE run is what keeps
+// this test honest about which one it took: an exclusive must be formed and no
+// [sets: ...] may survive. Without it, a builder change that stopped forming
+// the EXCLUSIVE would leave the sets to be locked at replay instead, and the
+// overlap count would stay 0 while the test silently stopped covering the
+// structural path.
+
+#include <atomic>
+#include <cstdio>
+#include <omp.h>
+
+#define ITERS 50
+
+// Number of tasks inside the critical section, and how often that exceeded one.
+static std::atomic<int> holders;
+static std::atomic<int> overlaps;
+
+// Stay in the critical section long enough that a concurrent holder is actually
+// observed rather than merely possible. A team too small to run the tasks
+// concurrently makes this vacuous, but never makes it fail.
+static void hold() {
+ if (holders.fetch_add(1) != 0)
+ overlaps.fetch_add(1);
+ double end = omp_get_wtime() + 0.0002;
+ while (omp_get_wtime() < end)
+ ;
+ holders.fetch_sub(1);
+}
+
+int main() {
+ int d;
+
+ for (int iter = 0; iter < ITERS; ++iter) {
+#pragma omp parallel num_threads(4)
+#pragma omp single
+ {
+#pragma omp taskgraph
+ {
+ // clang-format off
+#pragma omp task depend(out : d)
+ {}
+#pragma omp task depend(mutexinoutset : d)
+ { hold(); }
+#pragma omp task depend(mutexinoutset : d)
+ { hold(); }
+#pragma omp task depend(in : d)
+ {}
+ // clang-format on
+ }
+ }
+ }
+
+ std::printf("exclusive-region overlaps: %d\n", overlaps.load());
+ return overlaps.load() != 0;
+}
+
+// SHAPE: exclusive {
+
+// CHECK: exclusive-region overlaps: 0
diff --git a/openmp/runtime/test/taskgraph/taskgraph_mutex_irreducible_locks.cpp b/openmp/runtime/test/taskgraph/taskgraph_mutex_irreducible_locks.cpp
new file mode 100644
index 00000000000000..f1c6be3b02fe61
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_mutex_irreducible_locks.cpp
@@ -0,0 +1,112 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+// RUN: env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck --check-prefix=SHAPE --implicit-check-not='exclusive' %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Surviving mutex sets reached through real dependency edges, rather than the
+// flat all-siblings shape of taskgraph_mutex_runtime_locks.cpp.
+//
+// T1 (out: a) T2 (out: b)
+// | \ / |
+// | \ / |
+// TA{m0} TAB{m0,m1} TB{m1}
+// | | |
+// '------ T6 (in: x, y, z) --'
+//
+// The three mutex tasks have *different* predecessor sets (a, b, and both), so
+// the twin-merge pass cannot fold them into a single EXCLUSIVE the way it does
+// when they are twins, and their sets survive into replay. The differing
+// predecessors also make this shape non-series-parallel, so the members land in
+// an IRREDUCIBLE -- which is what the SHAPE run pins, together with the sets
+// surviving and no exclusive being formed.
+//
+// So this covers a lock taken part-way through the descriptor walk rather than
+// at the root, and covers it inside a knot: the mutex sets have to survive
+// __kmp_taskgraph_gather_mutex_sets descending into the IRREDUCIBLE for the
+// bits to be there to lock at all.
+//
+// The data checks are not incidental -- they confirm the ordering the mutex
+// tasks do have (after T1/T2, before T6) is still enforced, so a lock cannot be
+// "fixed" by serialising the graph and losing the dependency structure.
+//
+// This does discriminate: suppressing the two replay-time lock calls reports
+// ~100 overlaps here, so a pass is evidence the locks ran rather than evidence
+// the detector missed. A team too small to run the tasks concurrently makes
+// the check vacuous, but never makes it fail.
+
+#include <atomic>
+#include <cstdio>
+#include <omp.h>
+
+#define ITERS 50
+
+// Non-zero only if two tasks whose mutex sets intersect were inside their
+// critical sections at the same moment.
+static std::atomic<int> holders[2];
+static std::atomic<int> overlaps;
+static int errors;
+
+// Stay in the critical section long enough that a concurrent holder is actually
+// observed rather than merely possible. Pass r1 < 0 to hold just one.
+static void hold(int r0, int r1) {
+ if (holders[r0].fetch_add(1) != 0)
+ overlaps.fetch_add(1);
+ if (r1 >= 0 && holders[r1].fetch_add(1) != 0)
+ overlaps.fetch_add(1);
+ double end = omp_get_wtime() + 0.0002;
+ while (omp_get_wtime() < end)
+ ;
+ if (r1 >= 0)
+ holders[r1].fetch_sub(1);
+ holders[r0].fetch_sub(1);
+}
+
+int main() {
+ int a, b, m0, m1, x, y, z;
+
+ for (int iter = 0; iter < ITERS; ++iter) {
+#pragma omp parallel num_threads(4)
+#pragma omp single
+ {
+#pragma omp taskgraph
+ {
+ // clang-format off
+#pragma omp task depend(out : a)
+ { a = 1; }
+#pragma omp task depend(out : b)
+ { b = 2; }
+#pragma omp task depend(in : a) depend(mutexinoutset : m0) depend(out : x)
+ {
+ hold(0, -1);
+ x = a;
+ }
+#pragma omp task depend(in : b) depend(mutexinoutset : m1) depend(out : y)
+ {
+ hold(1, -1);
+ y = b;
+ }
+#pragma omp task depend(in : a, b) depend(mutexinoutset : m0, m1) depend(out : z)
+ {
+ hold(0, 1);
+ z = a + b;
+ }
+ // clang-format on
+#pragma omp task depend(in : x, y, z)
+ {
+ if (x != 1 || y != 2 || z != 3)
+ ++errors;
+ }
+ }
+ }
+ }
+
+ std::printf("irreducible mutex overlaps: %d, errors: %d\n", overlaps.load(),
+ errors);
+ return overlaps.load() != 0 || errors != 0;
+}
+
+// SHAPE: irreducible {
+// SHAPE: sets:
+
+// CHECK: irreducible mutex overlaps: 0, errors: 0
diff --git a/openmp/runtime/test/taskgraph/taskgraph_mutex_runtime_locks.cpp b/openmp/runtime/test/taskgraph/taskgraph_mutex_runtime_locks.cpp
new file mode 100644
index 00000000000000..1b0570eb385327
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_mutex_runtime_locks.cpp
@@ -0,0 +1,96 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+// RUN: env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck --check-prefix=SHAPE --implicit-check-not='exclusive' %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Mutual exclusion for a mutexinoutset that the region builder cannot express
+// structurally, so it has to be enforced by a lock during replay.
+//
+// The mutex sets overlap in a chain rather than partitioning into cliques:
+//
+// TA { d0 } TB { d1 } TAB { d0, d1 }
+//
+// TAB conflicts with both TA and TB, but TA and TB do not conflict with each
+// other, so no grouping of the three into EXCLUSIVE regions expresses exactly
+// this relation -- __kmp_taskgraph_find_exclusive_regions forms none, and
+// __kmp_taskgraph_strip_mutex_sets keeps the sets, counting them into
+// taskgraph->num_mutexes. The SHAPE run pins that: no exclusive is formed and
+// [sets: ...] survives on the nodes, so this really is the lock path and not
+// the structural one covered by taskgraph_mutex_exclusive_serialized.cpp.
+//
+// The set is duplicated over d2/d3 so the graph matches taskgraph_deps_13.cpp,
+// whose trace already records that these sets survive the builder.
+//
+// Nothing here constrains the order the conflicting tasks run in, only that
+// they do not overlap: the recording pass gets that from the normal dependency
+// machinery, and replay has to reproduce it via
+// __kmp_taskgraph_acquire_locks / __kmp_taskgraph_release_locks.
+//
+// This does discriminate: suppressing those two calls reports ~100 overlaps out
+// of 300 critical sections, and a variant with no dependencies at all reports
+// ~160, so the detector is not merely failing to catch a violation. A team too
+// small to run the tasks concurrently makes the check vacuous, but never makes
+// it fail.
+
+#include <atomic>
+#include <cstdio>
+#include <omp.h>
+
+#define ITERS 50
+#define NRES 4
+
+// Non-zero only if two tasks holding the same resource were inside their
+// critical sections at the same moment.
+static std::atomic<int> holders[NRES];
+static std::atomic<int> overlaps;
+
+// Stay in the critical section long enough that a concurrent holder is actually
+// observed rather than merely possible. Pass res1 < 0 to hold just one.
+static void hold(int res0, int res1) {
+ if (holders[res0].fetch_add(1) != 0)
+ overlaps.fetch_add(1);
+ if (res1 >= 0 && holders[res1].fetch_add(1) != 0)
+ overlaps.fetch_add(1);
+ double end = omp_get_wtime() + 0.0002;
+ while (omp_get_wtime() < end)
+ ;
+ if (res1 >= 0)
+ holders[res1].fetch_sub(1);
+ holders[res0].fetch_sub(1);
+}
+
+int main() {
+ int d[NRES];
+
+ for (int iter = 0; iter < ITERS; ++iter) {
+#pragma omp parallel num_threads(4)
+#pragma omp single
+ {
+#pragma omp taskgraph
+ {
+ // clang-format off
+#pragma omp task depend(mutexinoutset : d[0])
+ { hold(0, -1); }
+#pragma omp task depend(mutexinoutset : d[1])
+ { hold(1, -1); }
+#pragma omp task depend(mutexinoutset : d[0], d[1])
+ { hold(0, 1); }
+#pragma omp task depend(mutexinoutset : d[2])
+ { hold(2, -1); }
+#pragma omp task depend(mutexinoutset : d[3])
+ { hold(3, -1); }
+#pragma omp task depend(mutexinoutset : d[2], d[3])
+ { hold(2, 3); }
+ // clang-format on
+ }
+ }
+ }
+
+ std::printf("runtime-lock overlaps: %d\n", overlaps.load());
+ return overlaps.load() != 0;
+}
+
+// SHAPE: sets:
+
+// CHECK: runtime-lock overlaps: 0
diff --git a/openmp/runtime/test/taskgraph/taskgraph_record_serial_replay_parallel.cpp b/openmp/runtime/test/taskgraph/taskgraph_record_serial_replay_parallel.cpp
new file mode 100644
index 00000000000000..f47c27c139fcae
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_record_serial_replay_parallel.cpp
@@ -0,0 +1,50 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A graph recorded at top level, where the team is serialized, and replayed
+// from inside a parallel region has to run its tasks as tasks of the replaying
+// team. Were the recorded ones to keep the serialized flags they were created
+// with, the tasks they create would skip dependence tracking -- so the last
+// task below could run before the two it depends on -- and task memory would
+// go back to the wrong thread's pool.
+
+#include <stdio.h>
+
+static void shape(int *Out) {
+ int G1 = 0, G2 = 0, A = 0, B = 0;
+#pragma omp taskgraph
+ {
+#pragma omp task shared(A, B, G1, G2) firstprivate(Out)
+ {
+#pragma omp taskgroup
+ {
+#pragma omp task shared(A) depend(out : G1)
+ A = 1;
+#pragma omp task shared(B) depend(out : G2)
+ B = 2;
+#pragma omp task shared(A, B) firstprivate(Out) depend(in : G1, G2)
+ *Out += A + B;
+ }
+ }
+ }
+}
+
+int main() {
+ int Out = 0;
+ for (int It = 0; It < 3; ++It)
+ shape(&Out);
+ // CHECK: serialized: 9
+ printf("serialized: %d\n", Out);
+
+ Out = 0;
+#pragma omp parallel num_threads(4)
+#pragma omp single
+ for (int It = 0; It < 3; ++It)
+ shape(&Out);
+ // CHECK: parallel: 9
+ printf("parallel: %d\n", Out);
+ return Out != 9;
+}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_reduction_join.cpp b/openmp/runtime/test/taskgraph/taskgraph_reduction_join.cpp
new file mode 100644
index 00000000000000..155a5c60d6185b
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_reduction_join.cpp
@@ -0,0 +1,53 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A taskloop reduction's result is only combined into the original variable
+// when the taskloop's implicit taskgroup ends. On replay, a task that depends
+// on the reduction must therefore not start until that combine has happened,
+// not merely once the reduction's member tasks have finished. The second
+// taskloop reads 'res' through a shared reference; if it is released as soon
+// as the first taskloop's chunks complete it sees 0 instead of 45.
+
+#include <cstdio>
+
+int main() {
+ int arr[10], arr2[10];
+ for (int i = 0; i < 10; i++) {
+ arr[i] = i;
+ arr2[i] = 3 + i * 2;
+ }
+
+ int bad = 0;
+#pragma omp parallel
+#pragma omp single
+ for (int i = 0; i < 4; i++) {
+ int res = 0, res2 = 0, seen = -1;
+#pragma omp taskgraph graph_id(9107)
+ {
+#pragma omp taskloop reduction(+ : res) num_tasks(10)
+ for (int j = 0; j < 10; j++)
+ res += arr[j];
+#pragma omp taskloop reduction(+ : res2) num_tasks(10) shared(res, seen)
+ for (int j = 0; j < 10; j++) {
+ res2 += res * arr2[j];
+ if (j == 0)
+ seen = res;
+ }
+ }
+ if (res != 45 || res2 != 5400 || seen != 45) {
+ std::fprintf(stderr, "FAIL iter %d res=%d res2=%d seen=%d\n", i, res,
+ res2, seen);
+ bad = 1;
+ }
+ }
+
+ if (bad)
+ return 1;
+ std::fprintf(stderr, "PASS reduction join\n");
+ return 0;
+}
+
+// CHECK: PASS reduction join
diff --git a/openmp/runtime/test/taskgraph/taskgraph_reset_during_replay.cpp b/openmp/runtime/test/taskgraph/taskgraph_reset_during_replay.cpp
new file mode 100644
index 00000000000000..d0132a00a7f610
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_reset_during_replay.cpp
@@ -0,0 +1,94 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_CHECK_LIFETIME=1 %libomp-run 2>&1 | FileCheck %s --implicit-check-not='still attached'
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// One thread resets a taskgraph while the others replay it. A replay holds
+// only the record's map_lock and a reset takes only the header lock, so the
+// reset used to hand the record to the expire path -- freeing its region tree,
+// exec descriptors and recorded clones -- while a replay was still walking
+// them. That crashes or hangs.
+//
+// Two things this test has to get right, both of which quietly defeat an
+// earlier attempt:
+//
+// - The reset and the replays must come from the *same* directive site.
+// tdg_handle is per site, so two textual `taskgraph graph_id(1)`
+// directives are separate headers with separate record lists and never
+// interact, no matter how the graph ids line up. Hence the runtime
+// graph_reset expression below rather than two functions.
+// - The result values prove nothing. A reset re-records into the record it
+// just freed, so the blocks come straight back off the allocator freelists
+// into structures of the same shape and an in-flight replay reads
+// plausible values; runs that were about to crash still report every
+// result correct. So the invariant is checked directly, via
+// KMP_TASKGRAPH_CHECK_LIFETIME: the runtime reports any record torn down
+// with a replay still attached, and the implicit-check-not above fails the
+// test if it ever does. Deferred teardowns are expected and are reported
+// in different words.
+//
+// The tasks spin so that a replay is in flight long enough for a reset to land
+// in the middle of one.
+
+#include <cstdio>
+#include <omp.h>
+
+static constexpr int Threads = 16;
+static constexpr int Reps = 200;
+static constexpr int NTask = 8;
+static constexpr long Work = 20000;
+
+static volatile long sink = 0;
+
+static long spin(long n) {
+ long acc = 0;
+ for (long i = 0; i < n; ++i)
+ acc += i ^ n;
+ return acc;
+}
+
+static void graph(int *out, int base, int reset) {
+#pragma omp taskgraph graph_id(1) graph_reset(reset)
+ {
+ for (int i = 0; i < NTask; ++i) {
+#pragma omp task shared(out, base)
+ {
+ sink += spin(Work);
+ out[i] = base + i;
+ }
+ }
+ }
+}
+
+int main() {
+ int seed[NTask];
+ int wrong = 0;
+
+ // Record it once, single-threaded, so that it is READY to replay below.
+#pragma omp parallel num_threads(1)
+ graph(seed, 0, 0);
+
+#pragma omp parallel num_threads(Threads) reduction(+ : wrong)
+ {
+ const int me = omp_get_thread_num();
+ int out[NTask];
+ for (int rep = 0; rep < Reps; ++rep) {
+ for (int i = 0; i < NTask; ++i)
+ out[i] = -1;
+ // Only thread 0 resets; a second resetter would trip the (legitimate)
+ // concurrent-re-record diagnostic instead of exercising this path.
+ graph(out, me * 100, me == 0);
+ for (int i = 0; i < NTask; ++i)
+ if (out[i] != me * 100 + i)
+ ++wrong;
+ }
+ }
+
+ // Reported for information only: a reset racing a replay leaves the two
+ // encounters unordered, so a replay of the graph as it was before the reset
+ // is a legal outcome. Surviving with the invariant intact is the point.
+ std::printf("done, %d unexpected values\n", wrong);
+ return 0;
+}
+
+// CHECK: done
diff --git a/openmp/runtime/test/taskgraph/taskgraph_reset_no_leak.cpp b/openmp/runtime/test/taskgraph/taskgraph_reset_no_leak.cpp
new file mode 100644
index 00000000000000..69de0b046975a6
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_reset_no_leak.cpp
@@ -0,0 +1,126 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental, linux
+// clang-format on
+
+// Every graph_reset tears the previous record down and records afresh, so the
+// teardown has to return everything the recording took. It did not: the
+// per-node region array is a single allocation whose elements after the first
+// are not on the chain either cleanup walk follows, so their edge lists were
+// never released, and the transient depnodes the build borrows to discover the
+// edges kept their successor lists. That leaked about a kilobyte per recorded
+// task per reset -- unbounded in any program that re-records in a loop.
+//
+// Measured as resident-set growth, because what is leaked is small blocks that
+// libomp's own allocator would otherwise recycle: the bug is invisible to a
+// malloc-level leak checker (the blocks are freed by the process's own
+// teardown) and only shows up as memory the running program never gets back.
+//
+// The graphs below cover the shapes that reach the different cleanup paths: a
+// wide fan with chained dependencies for the ordinary series/parallel collapse,
+// and a non-series-parallel knot for the irreducible carve, which is the one
+// case that deliberately keeps edge lists alive until teardown.
+//
+// Dependences are arranged so that every address read in an iteration is also
+// written in it. A read of an address that is never written is remembered by
+// the runtime until some task writes it, which grows without bound whether or
+// not a taskgraph is involved, and would swamp what this test measures.
+
+#include <atomic>
+#include <cstdio>
+#include <unistd.h>
+
+static long rss_kb() {
+ std::FILE *f = std::fopen("/proc/self/statm", "r");
+ if (!f)
+ return -1;
+ long pages_total = 0, pages_resident = 0;
+ int got = std::fscanf(f, "%ld %ld", &pages_total, &pages_resident);
+ std::fclose(f);
+ if (got != 2)
+ return -1;
+ return pages_resident * (long)(sysconf(_SC_PAGESIZE) / 1024);
+}
+
+static std::atomic<int> Counter{0};
+
+template <typename BodyTy> static void replay(int iters, BodyTy &&body) {
+ for (int iter = 0; iter < iters; ++iter)
+ body();
+}
+
+int main() {
+ // The pre-fix leak is ~1 kB per recorded task per reset, so the fan graph
+ // alone leaked ~80 MB over the measured loop. Anything under a few MB of
+ // growth means the teardown is returning what the recording took; the slack
+ // is for the allocator's own arena granularity, not for a per-reset leak.
+ constexpr int WarmupIters = 200;
+ constexpr int MeasuredIters = 4000;
+ constexpr long LimitKB = 8 * 1024;
+
+ int fan[16] = {0};
+ int knot[2] = {0};
+ long before = 0, after = 0;
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ auto fan_graph = [&]() {
+#pragma omp taskgraph graph_id(1) graph_reset(1)
+ {
+ for (int i = 0; i < 16; ++i) {
+#pragma omp task depend(inout : fan[i])
+ Counter.fetch_add(1, std::memory_order_relaxed);
+ }
+ }
+ };
+
+ // A -> {B, C}, {B, C} -> D, C -> E: not series-parallel, so the build
+ // carves it into an IRREDUCIBLE region whose children keep their edges.
+ auto knot_graph = [&]() {
+#pragma omp taskgraph graph_id(2) graph_reset(1)
+ {
+#pragma omp task depend(out : knot[0], knot[1])
+ Counter.fetch_add(1, std::memory_order_relaxed);
+#pragma omp task depend(inout : knot[0])
+ Counter.fetch_add(1, std::memory_order_relaxed);
+#pragma omp task depend(inout : knot[1])
+ Counter.fetch_add(1, std::memory_order_relaxed);
+#pragma omp task depend(in : knot[0], knot[1])
+ Counter.fetch_add(1, std::memory_order_relaxed);
+#pragma omp task depend(in : knot[1])
+ Counter.fetch_add(1, std::memory_order_relaxed);
+ }
+ };
+
+ replay(WarmupIters, fan_graph);
+ replay(WarmupIters, knot_graph);
+ before = rss_kb();
+
+ replay(MeasuredIters, fan_graph);
+ replay(MeasuredIters, knot_graph);
+ after = rss_kb();
+ }
+ }
+
+ if (before < 0 || after < 0) {
+ std::fprintf(stderr, "FAIL could not read resident set size\n");
+ return 1;
+ }
+
+ const long growth = after - before;
+ if (growth > LimitKB) {
+ std::fprintf(stderr,
+ "FAIL taskgraph reset leaks: resident set grew %ld kB over "
+ "%d resets (limit %ld kB)\n",
+ growth, 2 * MeasuredIters, LimitKB);
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS taskgraph reset growth %ld kB over %d resets\n",
+ growth, 2 * MeasuredIters);
+ return 0;
+}
+
+// CHECK: PASS taskgraph reset growth
diff --git a/openmp/runtime/test/taskgraph/taskgraph_untied_parts.cpp b/openmp/runtime/test/taskgraph/taskgraph_untied_parts.cpp
new file mode 100644
index 00000000000000..0283ab1c0628d7
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_untied_parts.cpp
@@ -0,0 +1,38 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A recorded untied task that reaches a task scheduling point runs in parts,
+// and each replay runs all of them, from the first: the part a replay starts
+// from is not the one the last replay finished in.
+
+#include <atomic>
+#include <cstdio>
+
+static std::atomic<int> Before{0}, After{0};
+
+static void run() {
+#pragma omp taskgraph graph_id(1)
+ {
+ for (int I = 0; I < 4; ++I) {
+#pragma omp task untied
+ {
+ ++Before;
+#pragma omp taskyield
+ ++After;
+ }
+ }
+ }
+}
+
+int main() {
+#pragma omp parallel
+#pragma omp single
+ for (int R = 0; R < 5; ++R)
+ run();
+ std::printf("before %d after %d\n", Before.load(), After.load());
+ return 0;
+}
+
+// CHECK: before 20 after 20
diff --git a/openmp/runtime/test/tasking/omp_taskgraph_irreducible.cpp b/openmp/runtime/test/tasking/omp_taskgraph_irreducible.cpp
new file mode 100644
index 00000000000000..36d50090f13655
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_taskgraph_irreducible.cpp
@@ -0,0 +1,84 @@
+// clang-format off
+// REQUIRES: omp_taskgraph_experimental
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run
+// clang-format on
+//
+// NxN lattice "wavefront": cell (i,j) depends on (i-1,j) and (i,j-1). This is
+// the canonical non-series-parallel (irreducible) dependence graph. It used to
+// livelock the taskgraph region builder, which tried to make the graph
+// reducible by node-splitting -- a rewrite that does not converge on
+// lattice-like graphs. The builder now carves the residual into a single
+// TASKGRAPH_REGION_IRREDUCIBLE region instead, which is guaranteed to
+// terminate.
+//
+// Each cell accumulates the number of monotone lattice paths reaching it
+// (val[i][j] == val[i-1][j] + val[i][j-1], with val[0][0] == 1), i.e. the
+// binomial C(i+j, i). N is kept small enough that the values fit in long long.
+#include <cassert>
+#include <iostream>
+
+#define N 12
+#define ITERS 20
+
+static long long val[N][N];
+static long long ref[N][N];
+
+int main() {
+ // Serial reference, computed with the same recurrence to avoid any
+ // closed-form overflow surprises.
+ for (int i = 0; i < N; ++i)
+ for (int j = 0; j < N; ++j)
+ ref[i][j] =
+ (i == 0 && j == 0)
+ ? 1
+ : ((i > 0 ? ref[i - 1][j] : 0) + (j > 0 ? ref[i][j - 1] : 0));
+
+ for (int iter = 0; iter < ITERS; ++iter) {
+ for (int i = 0; i < N; ++i)
+ for (int j = 0; j < N; ++j)
+ val[i][j] = -1;
+
+#pragma omp parallel
+#pragma omp single
+ {
+#pragma omp taskgraph
+ {
+ for (int i = 0; i < N; ++i)
+ for (int j = 0; j < N; ++j) {
+ long long *me = &val[i][j];
+ long long *up = (i > 0) ? &val[i - 1][j] : nullptr;
+ long long *left = (j > 0) ? &val[i][j - 1] : nullptr;
+ if (i == 0 && j == 0) {
+#pragma omp task depend(out : me[0])
+ {
+ *me = 1;
+ }
+ } else if (i == 0) {
+#pragma omp task depend(in : left[0]) depend(out : me[0])
+ {
+ *me = *left;
+ }
+ } else if (j == 0) {
+#pragma omp task depend(in : up[0]) depend(out : me[0])
+ {
+ *me = *up;
+ }
+ } else {
+#pragma omp task depend(in : up[0], left[0]) depend(out : me[0])
+ {
+ *me = *up + *left;
+ }
+ }
+ }
+ }
+ }
+
+ for (int i = 0; i < N; ++i)
+ for (int j = 0; j < N; ++j)
+ assert(val[i][j] == ref[i][j] && "wavefront value mismatch");
+ }
+
+ std::cout << "Passed" << std::endl;
+ return 0;
+}
+// CHECK: Passed
>From 4f5c14545bc9694a76e24926b8657481202ec2f6 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 27 May 2026 07:51:15 -0500
Subject: [PATCH 09/29] [Clang][OpenMP] Parse and serialise the 'saved'
firstprivate modifier
This patch adds support for the OpenMP 6.0 'saved' modifier that case
be specified on a 'firstprivate' clause -- parsing, serialization and
deserialization.
New tests have been added to cover the new functionality.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/200406
---
clang/include/clang/AST/OpenMPClause.h | 38 ++++-
.../clang/Basic/DiagnosticSemaKinds.td | 3 +
clang/include/clang/Basic/OpenMPKinds.def | 7 +
clang/include/clang/Basic/OpenMPKinds.h | 7 +
clang/include/clang/Sema/SemaOpenMP.h | 8 +-
clang/lib/AST/OpenMPClause.cpp | 22 ++-
clang/lib/Basic/OpenMPKinds.cpp | 17 +-
clang/lib/Parse/ParseOpenMP.cpp | 13 ++
clang/lib/Sema/SemaOpenMP.cpp | 44 +++++-
clang/lib/Sema/TreeTransform.h | 10 +-
clang/lib/Serialization/ASTReader.cpp | 3 +
clang/lib/Serialization/ASTWriter.cpp | 3 +
...taskgraph_firstprivate_saved_ast_print.cpp | 146 ++++++++++++++++++
.../taskgraph_firstprivate_saved_messages.cpp | 115 ++++++++++++++
14 files changed, 409 insertions(+), 27 deletions(-)
create mode 100644 clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
create mode 100644 clang/test/OpenMP/taskgraph_firstprivate_saved_messages.cpp
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 8df512dffed3a2..4a95dd3e98dab3 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -3898,17 +3898,31 @@ class OMPFirstprivateClause final
friend OMPVarListClause;
friend TrailingObjects;
+ /// Optional firstprivate modifier (e.g. 'saved'), if specified by the user.
+ OpenMPFirstprivateModifier FPKind = OMPC_FIRSTPRIVATE_unknown;
+ /// Optional location of the firstprivate modifier, if specified.
+ SourceLocation FPKindLoc;
+ /// Optional location of the ':' separating the modifier from the list.
+ SourceLocation ColonLoc;
+
/// Build clause with number of variables \a N.
///
/// \param StartLoc Starting location of the clause.
/// \param LParenLoc Location of '('.
/// \param EndLoc Ending location of the clause.
+ /// \param FPKind Firstprivate modifier (e.g. 'saved').
+ /// \param FPKindLoc Location of the firstprivate modifier, if any.
+ /// \param ColonLoc Location of the ':' symbol, if a modifier is used.
/// \param N Number of the variables in the clause.
OMPFirstprivateClause(SourceLocation StartLoc, SourceLocation LParenLoc,
- SourceLocation EndLoc, unsigned N)
+ SourceLocation EndLoc,
+ OpenMPFirstprivateModifier FPKind,
+ SourceLocation FPKindLoc, SourceLocation ColonLoc,
+ unsigned N)
: OMPVarListClause<OMPFirstprivateClause>(llvm::omp::OMPC_firstprivate,
StartLoc, LParenLoc, EndLoc, N),
- OMPClauseWithPreInit(this) {}
+ OMPClauseWithPreInit(this), FPKind(FPKind), FPKindLoc(FPKindLoc),
+ ColonLoc(ColonLoc) {}
/// Build an empty clause.
///
@@ -3919,6 +3933,13 @@ class OMPFirstprivateClause final
SourceLocation(), N),
OMPClauseWithPreInit(this) {}
+ /// Sets the firstprivate modifier kind.
+ void setKind(OpenMPFirstprivateModifier Kind) { FPKind = Kind; }
+ /// Sets the location of the firstprivate modifier.
+ void setKindLoc(SourceLocation Loc) { FPKindLoc = Loc; }
+ /// Sets the location of the ':' separator.
+ void setColonLoc(SourceLocation Loc) { ColonLoc = Loc; }
+
/// Sets the list of references to private copies with initializers for
/// new private variables.
/// \param VL List of references.
@@ -3959,12 +3980,16 @@ class OMPFirstprivateClause final
/// \param InitVL List of references to auto generated variables used for
/// initialization of a single array element. Used if firstprivate variable is
/// of array type.
+ /// \param FPKind Firstprivate modifier, e.g. 'saved'.
+ /// \param FPKindLoc Location of the firstprivate modifier, if any.
+ /// \param ColonLoc Location of the ':' symbol, if a modifier is used.
/// \param PreInit Statement that must be executed before entering the OpenMP
/// region with this clause.
static OMPFirstprivateClause *
Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
SourceLocation EndLoc, ArrayRef<Expr *> VL, ArrayRef<Expr *> PrivateVL,
- ArrayRef<Expr *> InitVL, Stmt *PreInit);
+ ArrayRef<Expr *> InitVL, OpenMPFirstprivateModifier FPKind,
+ SourceLocation FPKindLoc, SourceLocation ColonLoc, Stmt *PreInit);
/// Creates an empty clause with the place for \a N variables.
///
@@ -3972,6 +3997,13 @@ class OMPFirstprivateClause final
/// \param N The number of variables.
static OMPFirstprivateClause *CreateEmpty(const ASTContext &C, unsigned N);
+ /// Firstprivate modifier (e.g. 'saved' if specified, otherwise 'unknown').
+ OpenMPFirstprivateModifier getKind() const { return FPKind; }
+ /// Returns the location of the firstprivate modifier, if any.
+ SourceLocation getKindLoc() const { return FPKindLoc; }
+ /// Returns the location of the ':' symbol, if any.
+ SourceLocation getColonLoc() const { return ColonLoc; }
+
using private_copies_iterator = MutableArrayRef<Expr *>::iterator;
using private_copies_const_iterator = ArrayRef<const Expr *>::iterator;
using private_copies_range = llvm::iterator_range<private_copies_iterator>;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 101c6482078935..878c28cd256300 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12802,6 +12802,9 @@ def err_omp_one_defaultmap_each_category: Error<
def err_omp_lastprivate_conditional_non_scalar : Error<
"expected list item of scalar type in 'lastprivate' clause with 'conditional' modifier"
>;
+def err_omp_firstprivate_saved_wrong_directive : Error<
+ "'saved' modifier on 'firstprivate' clause is only allowed on a 'task', "
+ "'taskloop', or 'target' construct">;
def err_omp_flush_order_clause_and_list : Error<
"'flush' directive with memory order clause '%0' cannot have the list">;
def note_omp_flush_order_clause_here : Note<
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 5b5222a2ab4463..1ed1c6c3121609 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -59,6 +59,9 @@
#ifndef OPENMP_DEVICE_TYPE_KIND
#define OPENMP_DEVICE_TYPE_KIND(Name)
#endif
+#ifndef OPENMP_FIRSTPRIVATE_KIND
+#define OPENMP_FIRSTPRIVATE_KIND(Name)
+#endif
#ifndef OPENMP_LASTPRIVATE_KIND
#define OPENMP_LASTPRIVATE_KIND(Name)
#endif
@@ -230,6 +233,9 @@ OPENMP_DEVICE_TYPE_KIND(host)
OPENMP_DEVICE_TYPE_KIND(nohost)
OPENMP_DEVICE_TYPE_KIND(any)
+// Type of the 'firstprivate' clause.
+OPENMP_FIRSTPRIVATE_KIND(saved)
+
// Type of the 'lastprivate' clause.
OPENMP_LASTPRIVATE_KIND(conditional)
@@ -320,6 +326,7 @@ OPENMP_USE_DEVICE_PTR_FALLBACK_MODIFIER(fb_preserve)
#undef OPENMP_ORIGINAL_SHARING_MODIFIER
#undef OPENMP_ORDER_KIND
#undef OPENMP_ORDER_MODIFIER
+#undef OPENMP_FIRSTPRIVATE_KIND
#undef OPENMP_LASTPRIVATE_KIND
#undef OPENMP_DEVICE_TYPE_KIND
#undef OPENMP_LINEAR_KIND
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index 5e94b4fa582b27..3fa09606ac41f7 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -161,6 +161,13 @@ enum OpenMPDeviceType {
OMPC_DEVICE_TYPE_unknown
};
+/// OpenMP 'firstprivate' clause modifier.
+enum OpenMPFirstprivateModifier {
+#define OPENMP_FIRSTPRIVATE_KIND(Name) OMPC_FIRSTPRIVATE_##Name,
+#include "clang/Basic/OpenMPKinds.def"
+ OMPC_FIRSTPRIVATE_unknown,
+};
+
/// OpenMP 'lastprivate' clause modifier.
enum OpenMPLastprivateModifier {
#define OPENMP_LASTPRIVATE_KIND(Name) OMPC_LASTPRIVATE_##Name,
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index e855b2a5eea6be..ec805132e2f5da 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1307,10 +1307,10 @@ class SemaOpenMP : public SemaBase {
SourceLocation LParenLoc,
SourceLocation EndLoc);
/// Called on well-formed 'firstprivate' clause.
- OMPClause *ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
- SourceLocation StartLoc,
- SourceLocation LParenLoc,
- SourceLocation EndLoc);
+ OMPClause *ActOnOpenMPFirstprivateClause(
+ ArrayRef<Expr *> VarList, OpenMPFirstprivateModifier FPKind,
+ SourceLocation FPKindLoc, SourceLocation ColonLoc,
+ SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc);
/// Called on well-formed 'lastprivate' clause.
OMPClause *ActOnOpenMPLastprivateClause(
ArrayRef<Expr *> VarList, OpenMPLastprivateModifier LPKind,
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 1d18473a3be405..b6bc401db5c14b 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -494,14 +494,14 @@ void OMPFirstprivateClause::setInits(ArrayRef<Expr *> VL) {
llvm::copy(VL, getPrivateCopies().end());
}
-OMPFirstprivateClause *
-OMPFirstprivateClause::Create(const ASTContext &C, SourceLocation StartLoc,
- SourceLocation LParenLoc, SourceLocation EndLoc,
- ArrayRef<Expr *> VL, ArrayRef<Expr *> PrivateVL,
- ArrayRef<Expr *> InitVL, Stmt *PreInit) {
+OMPFirstprivateClause *OMPFirstprivateClause::Create(
+ const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
+ SourceLocation EndLoc, ArrayRef<Expr *> VL, ArrayRef<Expr *> PrivateVL,
+ ArrayRef<Expr *> InitVL, OpenMPFirstprivateModifier FPKind,
+ SourceLocation FPKindLoc, SourceLocation ColonLoc, Stmt *PreInit) {
void *Mem = C.Allocate(totalSizeToAlloc<Expr *>(3 * VL.size()));
- OMPFirstprivateClause *Clause =
- new (Mem) OMPFirstprivateClause(StartLoc, LParenLoc, EndLoc, VL.size());
+ OMPFirstprivateClause *Clause = new (Mem) OMPFirstprivateClause(
+ StartLoc, LParenLoc, EndLoc, FPKind, FPKindLoc, ColonLoc, VL.size());
Clause->setVarRefs(VL);
Clause->setPrivateCopies(PrivateVL);
Clause->setInits(InitVL);
@@ -2617,7 +2617,13 @@ void OMPClausePrinter::VisitOMPPrivateClause(OMPPrivateClause *Node) {
void OMPClausePrinter::VisitOMPFirstprivateClause(OMPFirstprivateClause *Node) {
if (!Node->varlist_empty()) {
OS << "firstprivate";
- VisitOMPClauseList(Node, '(');
+ OpenMPFirstprivateModifier FPKind = Node->getKind();
+ if (FPKind != OMPC_FIRSTPRIVATE_unknown) {
+ OS << "("
+ << getOpenMPSimpleClauseTypeName(OMPC_firstprivate, Node->getKind())
+ << ":";
+ }
+ VisitOMPClauseList(Node, FPKind == OMPC_FIRSTPRIVATE_unknown ? '(' : ' ');
OS << ")";
}
}
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 77c383d53e2d30..1c9b05f16817bc 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -149,6 +149,11 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
#define OPENMP_SEVERITY_KIND(Name) .Case(#Name, OMPC_SEVERITY_##Name)
#include "clang/Basic/OpenMPKinds.def"
.Default(OMPC_SEVERITY_unknown);
+ case OMPC_firstprivate:
+ return llvm::StringSwitch<OpenMPFirstprivateModifier>(Str)
+#define OPENMP_FIRSTPRIVATE_KIND(Name) .Case(#Name, OMPC_FIRSTPRIVATE_##Name)
+#include "clang/Basic/OpenMPKinds.def"
+ .Default(OMPC_FIRSTPRIVATE_unknown);
case OMPC_lastprivate:
return llvm::StringSwitch<OpenMPLastprivateModifier>(Str)
#define OPENMP_LASTPRIVATE_KIND(Name) .Case(#Name, OMPC_LASTPRIVATE_##Name)
@@ -281,7 +286,6 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
case OMPC_allocator:
case OMPC_collapse:
case OMPC_private:
- case OMPC_firstprivate:
case OMPC_shared:
case OMPC_task_reduction:
case OMPC_in_reduction:
@@ -495,6 +499,16 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
#include "clang/Basic/OpenMPKinds.def"
}
llvm_unreachable("Invalid OpenMP 'severity' clause type");
+ case OMPC_firstprivate:
+ switch (Type) {
+ case OMPC_FIRSTPRIVATE_unknown:
+ return "unknown";
+#define OPENMP_FIRSTPRIVATE_KIND(Name) \
+ case OMPC_FIRSTPRIVATE_##Name: \
+ return #Name;
+#include "clang/Basic/OpenMPKinds.def"
+ }
+ llvm_unreachable("Invalid OpenMP 'firstprivate' clause type");
case OMPC_lastprivate:
switch (Type) {
case OMPC_LASTPRIVATE_unknown:
@@ -681,7 +695,6 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
case OMPC_allocator:
case OMPC_collapse:
case OMPC_private:
- case OMPC_firstprivate:
case OMPC_shared:
case OMPC_task_reduction:
case OMPC_in_reduction:
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 893bb06b749277..7a85330e3c940c 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -5308,6 +5308,19 @@ bool Parser::ParseOpenMPVarList(
<< "linear-modifier(list)" << getOpenMPClauseName(Kind)
<< "linear(list: [linear-modifier,] step(step-size))";
}
+ } else if (Kind == OMPC_firstprivate) {
+ // Try to parse modifier if any.
+ Data.ExtraModifier = OMPC_FIRSTPRIVATE_unknown;
+ // The 'saved' modifier is OpenMP 6.0+ only.
+ if (getLangOpts().OpenMP >= 60 && Tok.is(tok::identifier) &&
+ PP.LookAhead(0).is(tok::colon)) {
+ Data.ExtraModifier =
+ getOpenMPSimpleClauseType(Kind, PP.getSpelling(Tok), getLangOpts());
+ Data.ExtraModifierLoc = Tok.getLocation();
+ ConsumeToken();
+ assert(Tok.is(tok::colon) && "Expected colon.");
+ Data.ColonLoc = ConsumeToken();
+ }
} else if (Kind == OMPC_lastprivate) {
// Try to parse modifier if any.
Data.ExtraModifier = OMPC_LASTPRIVATE_unknown;
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 7214e2e7c50da2..52f510542cf69e 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6910,8 +6910,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
}
if (!ImpInfo.Firstprivates.empty()) {
if (OMPClause *Implicit = ActOnOpenMPFirstprivateClause(
- ImpInfo.Firstprivates.getArrayRef(), SourceLocation(),
- SourceLocation(), SourceLocation())) {
+ ImpInfo.Firstprivates.getArrayRef(), OMPC_FIRSTPRIVATE_unknown,
+ /*FPKindLoc=*/SourceLocation(), /*ColonLoc=*/SourceLocation(),
+ /*StartLoc=*/SourceLocation(), /*LParenLoc=*/SourceLocation(),
+ /*EndLoc=*/SourceLocation())) {
ClausesWithImplicit.push_back(Implicit);
ErrorFound = cast<OMPFirstprivateClause>(Implicit)->varlist_size() !=
ImpInfo.Firstprivates.size();
@@ -20937,7 +20939,11 @@ OMPClause *SemaOpenMP::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
Res = ActOnOpenMPPrivateClause(VarList, StartLoc, LParenLoc, EndLoc);
break;
case OMPC_firstprivate:
- Res = ActOnOpenMPFirstprivateClause(VarList, StartLoc, LParenLoc, EndLoc);
+ assert(0 <= ExtraModifier && ExtraModifier <= OMPC_FIRSTPRIVATE_unknown &&
+ "Unexpected firstprivate modifier.");
+ Res = ActOnOpenMPFirstprivateClause(
+ VarList, static_cast<OpenMPFirstprivateModifier>(ExtraModifier),
+ ExtraModifierLoc, ColonLoc, StartLoc, LParenLoc, EndLoc);
break;
case OMPC_lastprivate:
assert(0 <= ExtraModifier && ExtraModifier <= OMPC_LASTPRIVATE_unknown &&
@@ -21347,10 +21353,33 @@ OMPClause *SemaOpenMP::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
Vars, PrivateCopies);
}
-OMPClause *SemaOpenMP::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
- SourceLocation StartLoc,
- SourceLocation LParenLoc,
- SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPFirstprivateClause(
+ ArrayRef<Expr *> VarList, OpenMPFirstprivateModifier FPKind,
+ SourceLocation FPKindLoc, SourceLocation ColonLoc, SourceLocation StartLoc,
+ SourceLocation LParenLoc, SourceLocation EndLoc) {
+ if (FPKind == OMPC_FIRSTPRIVATE_unknown && FPKindLoc.isValid()) {
+ assert(ColonLoc.isValid() && "Colon location must be valid.");
+ Diag(FPKindLoc, diag::err_omp_unexpected_clause_value)
+ << getListOfPossibleValues(OMPC_firstprivate, /*First=*/0,
+ /*Last=*/OMPC_FIRSTPRIVATE_unknown)
+ << getOpenMPClauseNameForDiag(OMPC_firstprivate);
+ return nullptr;
+ }
+
+ // OpenMP 6.0 [7.2]: the 'saved' modifier has effect only on a clause
+ // that appears on a replayable construct. Per [14.6] the directives
+ // that admit a 'replayable' clause and that also admit 'firstprivate'
+ // are 'task', 'taskloop', and 'target' (the other replayable-eligible
+ // directives -- 'target_enter_data' etc. -- do not admit
+ // 'firstprivate' at all). Reject 'saved' on any other directive, as
+ // it can never become a replayable construct.
+ OpenMPDirectiveKind CurDir = DSAStack->getCurrentDirective();
+ if (FPKind == OMPC_FIRSTPRIVATE_saved && !isOpenMPTaskingDirective(CurDir) &&
+ !isOpenMPTargetExecutionDirective(CurDir)) {
+ Diag(FPKindLoc, diag::err_omp_firstprivate_saved_wrong_directive);
+ return nullptr;
+ }
+
SmallVector<Expr *, 8> Vars;
SmallVector<Expr *, 8> PrivateCopies;
SmallVector<Expr *, 8> Inits;
@@ -21672,6 +21701,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
return OMPFirstprivateClause::Create(
getASTContext(), StartLoc, LParenLoc, EndLoc, Vars, PrivateCopies, Inits,
+ FPKind, FPKindLoc, ColonLoc,
buildPreInits(getASTContext(), ExprCaptures));
}
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index da158f2791f0c3..ab120180812e34 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1960,11 +1960,14 @@ class TreeTransform {
/// By default, performs semantic analysis to build the new OpenMP clause.
/// Subclasses may override this routine to provide different behavior.
OMPClause *RebuildOMPFirstprivateClause(ArrayRef<Expr *> VarList,
+ OpenMPFirstprivateModifier FPKind,
+ SourceLocation FPKindLoc,
+ SourceLocation ColonLoc,
SourceLocation StartLoc,
SourceLocation LParenLoc,
SourceLocation EndLoc) {
- return getSema().OpenMP().ActOnOpenMPFirstprivateClause(VarList, StartLoc,
- LParenLoc, EndLoc);
+ return getSema().OpenMP().ActOnOpenMPFirstprivateClause(
+ VarList, FPKind, FPKindLoc, ColonLoc, StartLoc, LParenLoc, EndLoc);
}
/// Build a new OpenMP 'lastprivate' clause.
@@ -11676,7 +11679,8 @@ OMPClause *TreeTransform<Derived>::TransformOMPFirstprivateClause(
Vars.push_back(EVar.get());
}
return getDerived().RebuildOMPFirstprivateClause(
- Vars, C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
+ Vars, C->getKind(), C->getKindLoc(), C->getColonLoc(), C->getBeginLoc(),
+ C->getLParenLoc(), C->getEndLoc());
}
template <typename Derived>
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 7588b3fa61aef9..0840cd1066599c 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12429,6 +12429,9 @@ void OMPClauseReader::VisitOMPPrivateClause(OMPPrivateClause *C) {
void OMPClauseReader::VisitOMPFirstprivateClause(OMPFirstprivateClause *C) {
VisitOMPClauseWithPreInit(C);
C->setLParenLoc(Record.readSourceLocation());
+ C->setKind(Record.readEnum<OpenMPFirstprivateModifier>());
+ C->setKindLoc(Record.readSourceLocation());
+ C->setColonLoc(Record.readSourceLocation());
unsigned NumVars = C->varlist_size();
SmallVector<Expr *, 16> Vars;
Vars.reserve(NumVars);
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index cfcfbaa763b3b0..8c0845406b5b6f 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8455,6 +8455,9 @@ void OMPClauseWriter::VisitOMPFirstprivateClause(OMPFirstprivateClause *C) {
Record.push_back(C->varlist_size());
VisitOMPClauseWithPreInit(C);
Record.AddSourceLocation(C->getLParenLoc());
+ Record.writeEnum(C->getKind());
+ Record.AddSourceLocation(C->getKindLoc());
+ Record.AddSourceLocation(C->getColonLoc());
for (auto *VE : C->varlist()) {
Record.AddStmt(VE);
}
diff --git a/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp b/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
new file mode 100644
index 00000000000000..23cfbdddb3f68a
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
@@ -0,0 +1,146 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print %s | FileCheck %s
+
+#ifndef HEADER
+#define HEADER
+
+void firstprivate_saved() {
+ int a = 1;
+ int b = 2;
+ int c = 3;
+
+ // CHECK: #pragma omp task firstprivate(a)
+ #pragma omp task firstprivate(a)
+ { (void)a; }
+
+ // 'saved' on an omp task lexically inside a taskgraph.
+ // CHECK: #pragma omp taskgraph
+ // CHECK: #pragma omp task firstprivate(saved: a)
+ #pragma omp taskgraph
+ {
+ #pragma omp task firstprivate(saved: a)
+ { (void)a; }
+ }
+
+ // Multiple variables.
+ // CHECK: #pragma omp taskgraph
+ // CHECK: #pragma omp task firstprivate(saved: a,b,c)
+ #pragma omp taskgraph
+ {
+ #pragma omp task firstprivate(saved: a, b, c)
+ { (void)a; (void)b; (void)c; }
+ }
+
+ // Mixed with another clause.
+ // CHECK: #pragma omp taskgraph
+ // CHECK: #pragma omp task firstprivate(saved: a) shared(b)
+ #pragma omp taskgraph
+ {
+ #pragma omp task firstprivate(saved: a) shared(b)
+ { (void)a; (void)b; }
+ }
+
+ // 'saved' on an omp taskloop lexically inside a taskgraph.
+ // CHECK: #pragma omp taskgraph
+ // CHECK: #pragma omp taskloop firstprivate(saved: a)
+ #pragma omp taskgraph
+ {
+ #pragma omp taskloop firstprivate(saved: a)
+ for (int i = 0; i < 4; ++i) (void)a;
+ }
+
+ // 'saved' on a replayable omp task outside any taskgraph - also legal.
+ // CHECK: #pragma omp task replayable firstprivate(saved: a)
+ #pragma omp task replayable firstprivate(saved: a)
+ { (void)a; }
+
+ // 'saved' on a replayable omp taskloop outside any taskgraph - also legal.
+ // CHECK: #pragma omp taskloop replayable firstprivate(saved: a)
+ #pragma omp taskloop replayable firstprivate(saved: a)
+ for (int i = 0; i < 4; ++i) (void)a;
+
+ // 'saved' on a non-lexically-nested task (dynamic nesting via a call into
+ // a function from a taskgraph region is the runtime use case) - we accept
+ // any task/taskloop construct since the static check cannot prove dynamic
+ // nesting.
+ // CHECK: #pragma omp task firstprivate(saved: a)
+ #pragma omp task firstprivate(saved: a)
+ { (void)a; }
+}
+
+// Per OpenMP 6.0 [14.3], a 'firstprivate' clause with the 'saved' modifier on
+// a replayable construct may include variables with static storage duration;
+// they are copied into the saved data environment of the taskgraph record.
+// This covers file-scope statics, static-local variables, static data
+// members, and const-qualified statics, all of which Sema accepts and Clang
+// codegen places into the per-task '.kmp_privates.t' tail struct.
+
+static int FileScopeStatic = 100;
+static const int FileScopeConstStatic = 200;
+
+struct WithStaticMember {
+ static int StaticMember;
+ static const int StaticConstMember = 400;
+};
+int WithStaticMember::StaticMember = 0;
+
+void firstprivate_saved_statics() {
+ static int LocalStatic = 300;
+ static const int LocalConstStatic = 500;
+
+ // CHECK-LABEL: void firstprivate_saved_statics
+ // CHECK: #pragma omp task firstprivate(saved: FileScopeStatic)
+ #pragma omp task firstprivate(saved: FileScopeStatic)
+ { (void)FileScopeStatic; }
+
+ // CHECK: #pragma omp task firstprivate(saved: FileScopeConstStatic)
+ #pragma omp task firstprivate(saved: FileScopeConstStatic)
+ { (void)FileScopeConstStatic; }
+
+ // CHECK: #pragma omp task firstprivate(saved: LocalStatic)
+ #pragma omp task firstprivate(saved: LocalStatic)
+ { (void)LocalStatic; }
+
+ // CHECK: #pragma omp task firstprivate(saved: LocalConstStatic)
+ #pragma omp task firstprivate(saved: LocalConstStatic)
+ { (void)LocalConstStatic; }
+
+ // CHECK: #pragma omp task firstprivate(saved: WithStaticMember::StaticMember)
+ #pragma omp task firstprivate(saved: WithStaticMember::StaticMember)
+ { (void)WithStaticMember::StaticMember; }
+
+ // CHECK: #pragma omp task firstprivate(saved: WithStaticMember::StaticConstMember)
+ #pragma omp task firstprivate(saved: WithStaticMember::StaticConstMember)
+ { (void)WithStaticMember::StaticConstMember; }
+
+ // Multiple statics in a single clause, mixed with a non-static.
+ int local_int = 0;
+ // CHECK: #pragma omp task firstprivate(saved: FileScopeStatic,LocalStatic,WithStaticMember::StaticMember,local_int)
+ #pragma omp task firstprivate(saved: \
+ FileScopeStatic, LocalStatic, \
+ WithStaticMember::StaticMember, local_int)
+ {
+ (void)FileScopeStatic;
+ (void)LocalStatic;
+ (void)WithStaticMember::StaticMember;
+ (void)local_int;
+ }
+
+ // Same on a 'taskloop' construct.
+ // CHECK: #pragma omp taskloop firstprivate(saved: FileScopeStatic,LocalConstStatic)
+ #pragma omp taskloop firstprivate(saved: FileScopeStatic, LocalConstStatic)
+ for (int i = 0; i < 4; ++i) {
+ (void)FileScopeStatic;
+ (void)LocalConstStatic;
+ }
+}
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_firstprivate_saved_messages.cpp b/clang/test/OpenMP/taskgraph_firstprivate_saved_messages.cpp
new file mode 100644
index 00000000000000..5197d16cd04215
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_firstprivate_saved_messages.cpp
@@ -0,0 +1,115 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+
+// Tests the OpenMP 6.0 'saved' modifier on the 'firstprivate' clause. The
+// modifier is meaningful only on constructs that create tasks or taskloops
+// (the units of work that can participate in taskgraph replay). Every other
+// directive that admits a 'firstprivate' clause must reject it.
+
+void unknown_modifier() {
+ int a = 0;
+ // The diagnostic comes from the generic "expected <list> in OpenMP clause"
+ // path and enumerates the legal modifier names ('saved' in OpenMP 6.0).
+ #pragma omp task firstprivate(bogus: a) // expected-error {{expected 'saved' in OpenMP clause 'firstprivate'}}
+ { (void)a; }
+}
+
+void rejected_on_non_tasking_constructs() {
+ int a = 0;
+ int b[8];
+
+ // parallel
+ #pragma omp parallel firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+ { (void)a; }
+
+ // for (worksharing)
+ #pragma omp parallel
+ {
+ #pragma omp for firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+ for (int i = 0; i < 4; ++i) (void)a;
+ }
+
+ // sections
+ #pragma omp parallel
+ {
+ #pragma omp sections firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+ {
+ (void)a;
+ }
+ }
+
+ // single
+ #pragma omp parallel
+ {
+ #pragma omp single firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+ { (void)a; }
+ }
+
+ // teams (inside target -- the inner directive is a standalone 'teams',
+ // not a 'target teams' combined directive, so it is not a target
+ // execution directive in its own right).
+ #pragma omp target
+ #pragma omp teams firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+ { (void)a; }
+
+ // distribute (inside teams) -- standalone 'distribute' is not a target
+ // execution directive either.
+ #pragma omp target teams
+ #pragma omp distribute firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+ for (int i = 0; i < 4; ++i) (void)a;
+}
+
+void accepted_on_task_taskloop_and_target() {
+ int a = 0;
+
+ // Bare task (no enclosing taskgraph, no replayable clause): accepted.
+ // Per OpenMP 6.0 [7.2] the 'saved' modifier silently has no effect on a
+ // non-replayable construct; we only enforce the directive-kind check
+ // statically. In this implementation a bare task / taskloop is never
+ // recorded into a taskgraph -- recording requires either lexical
+ // nesting inside a '#pragma omp taskgraph' or an explicit 'replayable'
+ // clause -- so the modifier is a well-defined no-op here.
+ #pragma omp task firstprivate(saved: a)
+ { (void)a; }
+
+ // Bare taskloop: accepted, same rationale.
+ #pragma omp taskloop firstprivate(saved: a)
+ for (int i = 0; i < 4; ++i) (void)a;
+
+ // Replayable task: explicitly opted-in for replay.
+ #pragma omp task replayable firstprivate(saved: a)
+ { (void)a; }
+
+ // Replayable taskloop.
+ #pragma omp taskloop replayable firstprivate(saved: a)
+ for (int i = 0; i < 4; ++i) (void)a;
+
+ // Task lexically nested inside a taskgraph.
+ #pragma omp taskgraph
+ {
+ #pragma omp task firstprivate(saved: a)
+ { (void)a; }
+ }
+
+ // Bare target: accepted on the same well-formed-but-no-effect grounds
+ // as a bare task. Per OpenMP 6.0 [14.6] the 'target' construct admits
+ // both 'firstprivate' and 'replayable', so 'saved' is meaningful as
+ // soon as a 'replayable' clause is added or the construct is nested
+ // inside a 'taskgraph' region.
+ #pragma omp target firstprivate(saved: a)
+ { (void)a; }
+
+ // Replayable target: explicitly opted-in for replay.
+ #pragma omp target replayable firstprivate(saved: a)
+ { (void)a; }
+
+ // Combined target construct (target + parallel): accepted because the
+ // composite directive is a target execution directive in its own
+ // right, so the captured snapshot at the target boundary belongs to a
+ // construct that may participate in taskgraph replay.
+ #pragma omp target parallel firstprivate(saved: a)
+ { (void)a; }
+
+ // Combined target teams distribute parallel for: same rationale.
+ #pragma omp target teams distribute parallel for firstprivate(saved: a)
+ for (int i = 0; i < 4; ++i) (void)a;
+}
>From 7b30668010d5d0cebb0e35a5a1b75e677f2db916 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 7 Oct 2026 14:35:01 -0500
Subject: [PATCH 10/29] [Clang][OpenMP] Have a replayable task's body construct
its own private copies
A task which has variables listed in firstprivate clauses will normally
initialise those as part of task initialisation, which is done once per
instantiation (and execution) of a task. So, in the non-taskgraph case,
a task is able to modify its copy of the firstprivate variable.
In the taskgraph case though, that causes a problem. We want the
task/taskdata structures to be retained as part of the taskgraph record
to avoid the overhead of reallocating them. But if a task mutates its
firstprivate data on one replay of the taskgraph, we do NOT want that
mutated value to persist into the next replay of that task.
This patch is a solution to that. The task structure now holds two
copies of the data: one holding the initial value, and one for use
during execution of the task. Potentially, these can be variables with
non-trivially-copyable types: as such, the task's working copy of the
variable is copy-constructed at task start, and destroyed at the task end.
All this only happens for tasks which are or might be replayable: the
common non-taskgraph case should not incur any new overhead.
Tests are added to cover the new functionality.
Assisted-By: Claude
---
clang/lib/CodeGen/CGOpenMPRuntime.cpp | 143 +++++++++++-------
clang/lib/CodeGen/CGOpenMPRuntime.h | 6 +
clang/lib/CodeGen/CGStmtOpenMP.cpp | 128 +++++++++++++++-
clang/lib/CodeGen/CodeGenFunction.h | 9 ++
.../taskgraph_task_body_copies_codegen.cpp | 107 +++++++++++++
.../taskgraph_firstprivate_body_writes.cpp | 40 +++++
6 files changed, 377 insertions(+), 56 deletions(-)
create mode 100644 clang/test/OpenMP/taskgraph_task_body_copies_codegen.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_body_writes.cpp
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 492344606c7f56..1ea11e6f1e3b23 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -3189,6 +3189,9 @@ struct PrivateHelpersTy {
const VarDecl *Original = nullptr;
const VarDecl *PrivateCopy = nullptr;
const VarDecl *PrivateElemInit = nullptr;
+ /// The body's own copy of a firstprivate, beside its initial value (see
+ /// OMPTaskDataTy::BodyConstructsPrivates).
+ bool WorkingCopy = false;
bool isLocalPrivate() const {
return !OriginalRef && !PrivateCopy && !PrivateElemInit;
}
@@ -3204,6 +3207,17 @@ static const VarDecl *getOriginalVarDecl(const ValueDecl *Decl) {
return cast<VarDecl>(Decl);
}
+/// Whether \p P is constructed with the task descriptor and destroyed with it.
+/// That is every copy but a private local of an untied task, unless the body
+/// constructs its own copies (OMPTaskDataTy::BodyConstructsPrivates), when it
+/// is only a firstprivate's initial value.
+static bool isDescriptorCopy(const PrivateHelpersTy &P,
+ const OMPTaskDataTy &Data) {
+ if (P.isLocalPrivate())
+ return false;
+ return !Data.BodyConstructsPrivates || (P.PrivateElemInit && !P.WorkingCopy);
+}
+
static bool isAllocatableDecl(const VarDecl *VD) {
const VarDecl *CVD = VD->getCanonicalDecl();
if (!CVD->hasAttr<OMPAllocateDeclAttr>())
@@ -3252,6 +3266,17 @@ createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) {
I != E; ++I)
FD->addAttr(*I);
}
+ // The body accesses its copy of a firstprivate as the variable, which
+ // can be aligned beyond its type.
+ if (Pair.second.WorkingCopy && Pair.first > C.getTypeAlignInChars(Type)) {
+ llvm::APInt Align(32, Pair.first.getQuantity());
+ FD->addAttr(AlignedAttr::CreateImplicit(
+ C, /*IsAlignmentExpr=*/true,
+ IntegerLiteral::Create(C, Align,
+ C.getIntTypeForBitwidth(32, /*Signed=*/0),
+ SourceLocation()),
+ {}, AlignedAttr::GNU_aligned));
+ }
}
RD->completeDefinition();
return RD;
@@ -3438,11 +3463,10 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
return TaskEntry;
}
-static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
- SourceLocation Loc,
- QualType KmpInt32Ty,
- QualType KmpTaskTWithPrivatesPtrQTy,
- QualType KmpTaskTWithPrivatesQTy) {
+static llvm::Value *emitDestructorsFunction(
+ CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty,
+ QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy,
+ const OMPTaskDataTy &Data, ArrayRef<PrivateDataTy> Privates) {
ASTContext &C = CGM.getContext();
auto *GtidArg =
ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
@@ -3476,7 +3500,12 @@ static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM,
KmpTaskTWithPrivatesQTy->castAsRecordDecl();
auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
Base = CGF.EmitLValueForField(Base, *FI);
+ auto PI = Privates.begin();
for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
+ const PrivateHelpersTy &P = (PI++)->second;
+ // The body destroys the copies it constructs.
+ if (Data.BodyConstructsPrivates && !isDescriptorCopy(P, Data))
+ continue;
if (QualType::DestructionKind DtorKind =
Field->getType().isDestructedType()) {
LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
@@ -3567,6 +3596,22 @@ emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
PrivateVarsPos[VD] = Counter;
++Counter;
}
+ // The body's own copies of the firstprivates, keyed by canonical declaration
+ // since a firstprivate may be a BindingDecl.
+ llvm::SmallDenseMap<const ValueDecl *, unsigned> WorkingCopyPos;
+ if (Data.BodyConstructsPrivates) {
+ for (const Expr *E : Data.FirstprivateVars) {
+ Args.push_back(ImplicitParamDecl::Create(
+ C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
+ C.getPointerType(C.getPointerType(E->getType()))
+ .withConst()
+ .withRestrict(),
+ ImplicitParamKind::Other));
+ const ValueDecl *VD = cast<DeclRefExpr>(E)->getDecl();
+ WorkingCopyPos[cast<ValueDecl>(VD->getCanonicalDecl())] = Counter;
+ ++Counter;
+ }
+ }
const auto &TaskPrivatesMapFnInfo =
CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
llvm::FunctionType *TaskPrivatesMapTy =
@@ -3599,12 +3644,12 @@ emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
for (const FieldDecl *Field : PrivatesQTyRD->fields()) {
LValue FieldLVal = CGF.EmitLValueForField(Base, Field);
// Lookup by the original declaration (BindingDecl or VarDecl).
+ const PrivateHelpersTy &P = Privates[Counter].second;
const ValueDecl *LookupVD;
- if (Privates[Counter].second.OriginalRef) {
- LookupVD =
- cast<DeclRefExpr>(Privates[Counter].second.OriginalRef)->getDecl();
+ if (P.OriginalRef) {
+ LookupVD = cast<DeclRefExpr>(P.OriginalRef)->getDecl();
} else {
- LookupVD = Privates[Counter].second.Original;
+ LookupVD = P.Original;
}
// For BindingDecls, the privates record now stores each binding's type
@@ -3612,7 +3657,11 @@ emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
// correct.
unsigned Position;
- if (const auto *BD = dyn_cast<BindingDecl>(LookupVD)) {
+ if (P.WorkingCopy) {
+ Position =
+ WorkingCopyPos.lookup(cast<ValueDecl>(LookupVD->getCanonicalDecl()));
+ assert(Position && "working copy not in privates mapping");
+ } else if (const auto *BD = dyn_cast<BindingDecl>(LookupVD)) {
Position =
BindingDeclPos.lookup(cast<BindingDecl>(BD->getCanonicalDecl()));
assert(Position && "binding not in privates mapping");
@@ -3664,8 +3713,8 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
}
FI = FI->getType()->castAsRecordDecl()->field_begin();
for (const PrivateDataTy &Pair : Privates) {
- // Do not initialize private locals.
- if (Pair.second.isLocalPrivate()) {
+ // Do not initialize the copies the body constructs.
+ if (!isDescriptorCopy(Pair.second, Data)) {
++FI;
continue;
}
@@ -3742,38 +3791,8 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
/*HasCancel=*/false, /*NoInheritance=*/true);
SharedRefLValue = CGF.EmitLValue(Pair.second.OriginalRef);
}
- if (Type->isArrayType()) {
- // Initialize firstprivate array.
- if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) {
- // Perform simple memcpy.
- CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type);
- } else {
- // Initialize firstprivate array using element-by-element
- // initialization.
- CGF.EmitOMPAggregateAssign(
- PrivateLValue.getAddress(), SharedRefLValue.getAddress(), Type,
- [&CGF, Elem, Init, &CapturesInfo](Address DestElement,
- Address SrcElement) {
- // Clean up any temporaries needed by the initialization.
- CodeGenFunction::OMPPrivateScope InitScope(CGF);
- InitScope.addPrivate(Elem, SrcElement);
- (void)InitScope.Privatize();
- // Emit initialization for single element.
- CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(
- CGF, &CapturesInfo);
- CGF.EmitAnyExprToMem(Init, DestElement,
- Init->getType().getQualifiers(),
- /*IsInitializer=*/false);
- });
- }
- } else {
- CodeGenFunction::OMPPrivateScope InitScope(CGF);
- InitScope.addPrivate(Elem, SharedRefLValue.getAddress());
- (void)InitScope.Privatize();
- CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);
- CGF.EmitExprAsInit(Init, VD, PrivateLValue,
- /*capturedByInit=*/false);
- }
+ CGF.EmitOMPPrivateCopyInit(Init, VD, Elem, PrivateLValue,
+ SharedRefLValue, &CapturesInfo);
} else {
CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false);
}
@@ -3850,10 +3869,11 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
/// Check if duplication function is required for taskloops.
static bool checkInitIsRequired(CodeGenFunction &CGF,
- ArrayRef<PrivateDataTy> Privates) {
+ ArrayRef<PrivateDataTy> Privates,
+ const OMPTaskDataTy &Data) {
bool InitRequired = false;
for (const PrivateDataTy &Pair : Privates) {
- if (Pair.second.isLocalPrivate())
+ if (!isDescriptorCopy(Pair.second, Data))
continue;
const VarDecl *VD = Pair.second.PrivateCopy;
const Expr *Init = VD->getAnyInitializer();
@@ -3865,7 +3885,6 @@ static bool checkInitIsRequired(CodeGenFunction &CGF,
return InitRequired;
}
-
/// Emit task_dup function (for initialization of
/// private/firstprivate/lastprivate vars and last_iter flag)
/// \code
@@ -3947,11 +3966,10 @@ emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
/// Checks if destructor function is required to be generated.
/// \return true if cleanups are required, false otherwise.
-static bool
-checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD,
- ArrayRef<PrivateDataTy> Privates) {
+static bool checkDestructorsRequired(ArrayRef<PrivateDataTy> Privates,
+ const OMPTaskDataTy &Data) {
for (const PrivateDataTy &P : Privates) {
- if (P.second.isLocalPrivate())
+ if (!isDescriptorCopy(P.second, Data))
continue;
QualType Ty = P.second.Original->getType().getNonReferenceType();
if (Ty.isDestructedType())
@@ -4116,6 +4134,21 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
++I;
++IElemInitRef;
}
+ // The body's own copies of the firstprivates, beside their initial values.
+ if (Data.BodyConstructsPrivates) {
+ I = Data.FirstprivateCopies.begin();
+ IElemInitRef = Data.FirstprivateInits.begin();
+ for (const Expr *E : Data.FirstprivateVars) {
+ const auto *VD = getOriginalVarDecl(cast<DeclRefExpr>(E)->getDecl());
+ PrivateHelpersTy Copy(
+ E, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
+ cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl()));
+ Copy.WorkingCopy = true;
+ Privates.emplace_back(C.getDeclAlign(VD), Copy);
+ ++I;
+ ++IElemInitRef;
+ }
+ }
I = Data.LastprivateCopies.begin();
for (const Expr *E : Data.LastprivateVars) {
const auto *Decl = cast<DeclRefExpr>(E)->getDecl();
@@ -4209,8 +4242,7 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
unsigned Flags = Data.Tied ? TiedFlag : 0;
bool NeedsCleanup = false;
if (!Privates.empty()) {
- NeedsCleanup =
- checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD, Privates);
+ NeedsCleanup = checkDestructorsRequired(Privates, Data);
if (NeedsCleanup)
Flags = Flags | DestructorsFlag;
}
@@ -4451,7 +4483,8 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
SharedsTy, SharedsPtrTy, Data, Privates,
/*ForDup=*/false);
if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
- (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
+ (!Data.LastprivateVars.empty() ||
+ checkInitIsRequired(CGF, Privates, Data))) {
Result.TaskDupFn = emitTaskDupFunction(
CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
@@ -4467,7 +4500,7 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
if (NeedsCleanup) {
llvm::Value *DestructorFn = emitDestructorsFunction(
CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
- KmpTaskTWithPrivatesQTy);
+ KmpTaskTWithPrivatesQTy, Data, Privates);
LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
LValue DestructorsLV = CGF.EmitLValueForField(
Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 4d9a69fa1f1d9f..01e520f7f40d7f 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -124,6 +124,12 @@ struct OMPTaskDataTy final {
bool HasNowaitClause = false;
bool HasModifier = false;
const Expr *ReplayableCond = nullptr;
+ /// The task may be recorded into a taskgraph, whose replays run its recorded
+ /// descriptor again, so running the body must leave the descriptor as it
+ /// was. The descriptor holds only the initial values of the firstprivates;
+ /// the body constructs its own copy of every private list item, from those
+ /// or by default initialization, and destroys it when the body ends.
+ bool BodyConstructsPrivates = false;
};
/// Class intended to support codegen of all kind of the reduction clauses.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index a062c86f080d89..62fe474624be88 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -1161,6 +1161,49 @@ void CodeGenFunction::EmitOMPAggregateAssign(
EmitBlock(DoneBB, /*IsFinished=*/true);
}
+void CodeGenFunction::EmitOMPPrivateCopyInit(const Expr *Init,
+ const VarDecl *PrivateCopy,
+ const VarDecl *ElemInit,
+ LValue Dst, LValue Src,
+ CGCapturedStmtInfo *CapturesInfo) {
+ if (!CapturesInfo)
+ CapturesInfo = CapturedStmtInfo;
+ if (!ElemInit) {
+ EmitExprAsInit(Init, PrivateCopy, Dst, /*capturedByInit=*/false);
+ return;
+ }
+ QualType Type = Dst.getType();
+ if (Type->isArrayType()) {
+ // Initialize firstprivate array.
+ if (!isa<CXXConstructExpr>(Init) || isTrivialInitializer(Init)) {
+ // Perform simple memcpy.
+ EmitAggregateAssign(Dst, Src, Type);
+ } else {
+ // Initialize firstprivate array using element-by-element
+ // initialization.
+ EmitOMPAggregateAssign(
+ Dst.getAddress(), Src.getAddress(), Type,
+ [this, ElemInit, Init, CapturesInfo](Address DestElement,
+ Address SrcElement) {
+ // Clean up any temporaries needed by the initialization.
+ OMPPrivateScope InitScope(*this);
+ InitScope.addPrivate(ElemInit, SrcElement);
+ (void)InitScope.Privatize();
+ // Emit initialization for single element.
+ CGCapturedStmtRAII CapInfoRAII(*this, CapturesInfo);
+ EmitAnyExprToMem(Init, DestElement, Init->getType().getQualifiers(),
+ /*IsInitializer=*/false);
+ });
+ }
+ return;
+ }
+ OMPPrivateScope InitScope(*this);
+ InitScope.addPrivate(ElemInit, Src.getAddress());
+ (void)InitScope.Privatize();
+ CGCapturedStmtRAII CapInfoRAII(*this, CapturesInfo);
+ EmitExprAsInit(Init, PrivateCopy, Dst, /*capturedByInit=*/false);
+}
+
void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
Address SrcAddr, const VarDecl *DestVD,
const VarDecl *SrcVD, const Expr *Copy) {
@@ -5544,6 +5587,13 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
Data.PrivateLocals.append(Checker.getPrivateDecls().begin(),
Checker.getPrivateDecls().end());
}
+ // A task that may be recorded into a taskgraph is replayed from its recorded
+ // descriptor, so its body works on copies of its own.
+ if (const Expr *Cond = Data.ReplayableCond) {
+ bool KnownValue;
+ Data.BodyConstructsPrivates =
+ !ConstantFoldsToSimpleInteger(Cond, KnownValue) || KnownValue;
+ }
auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
&LastprivateSrcsOrigs, CapturedRegion](
CodeGenFunction &CGF, PrePostActionTy &Action) {
@@ -5612,6 +5662,16 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
}
llvm::SmallVector<std::pair<const ValueDecl *, Address>, 16>
FirstprivatePtrs;
+ // The copies the body constructs itself (BodyConstructsPrivates), with
+ // the declaration whose initializer constructs each and, for a
+ // firstprivate, the element the initializer reads and its initial value.
+ struct BodyCopyTy {
+ const VarDecl *PrivateCopy;
+ const VarDecl *ElemInit;
+ LValue Dst;
+ LValue Src;
+ };
+ SmallVector<BodyCopyTy, 4> BodyCopies;
if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
!Data.LastprivateVars.empty() || !Data.PrivateLocals.empty()) {
enum { PrivatesParam = 2, CopyFnParam = 3 };
@@ -5638,7 +5698,9 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
CGF.getContext().getPointerType(E->getType()),
".firstpriv.ptr.addr");
- PrivatePtrs.emplace_back(VD, PrivatePtr);
+ // The body reads the initial value only to construct its own copy.
+ if (!Data.BodyConstructsPrivates)
+ PrivatePtrs.emplace_back(VD, PrivatePtr);
FirstprivatePtrs.emplace_back(VD, PrivatePtr);
CallArgs.push_back(PrivatePtr.getPointer());
ParamTypes.push_back(PrivatePtr.getType());
@@ -5669,6 +5731,18 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
CallArgs.push_back(PrivatePtr.getPointer());
ParamTypes.push_back(PrivatePtr.getType());
}
+ // The body's own copies of the firstprivates.
+ if (Data.BodyConstructsPrivates) {
+ for (const Expr *E : Data.FirstprivateVars) {
+ const auto *VD = cast<DeclRefExpr>(E)->getDecl();
+ RawAddress PrivatePtr = CGF.CreateMemTempWithoutCast(
+ CGF.getContext().getPointerType(E->getType()),
+ ".firstpriv.copy.ptr.addr");
+ PrivatePtrs.emplace_back(VD, PrivatePtr);
+ CallArgs.push_back(PrivatePtr.getPointer());
+ ParamTypes.push_back(PrivatePtr.getType());
+ }
+ }
auto *CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),
ParamTypes, /*isVarArg=*/false);
CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(
@@ -5690,6 +5764,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
Scope.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());
}
}
+ llvm::DenseMap<const ValueDecl *, Address> PrivateAddrs;
for (const auto &Pair : PrivatePtrs) {
Address Replacement = Address(
CGF.Builder.CreateLoad(Pair.second),
@@ -5707,6 +5782,7 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
}
}
}
+ PrivateAddrs.try_emplace(Pair.first, Replacement);
if (auto *DI = CGF.getDebugInfo())
if (CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())
@@ -5716,6 +5792,41 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
VD, Pair.second.getBasePointer(), CGF.Builder,
/*UsePointerValue*/ true);
}
+ if (Data.BodyConstructsPrivates) {
+ // A field of the privates record is aligned as its type is, which
+ // can be less than the variable's declared alignment.
+ auto AddCopy = [&](const Expr *Ref, const Expr *CopyRef,
+ const VarDecl *ElemInit, Address Src) {
+ const auto *VD = cast<DeclRefExpr>(Ref)->getDecl();
+ QualType Ty = VD->getType().getNonReferenceType();
+ CharUnits Align = CGF.getContext().getTypeAlignInChars(Ty);
+ LValue SrcLV;
+ if (Src.isValid())
+ SrcLV = CGF.MakeAddrLValue(Src.withAlignment(Align), Ty);
+ BodyCopies.push_back(
+ {cast<VarDecl>(cast<DeclRefExpr>(CopyRef)->getDecl()), ElemInit,
+ CGF.MakeAddrLValue(
+ PrivateAddrs.find(VD)->second.withAlignment(Align), Ty),
+ SrcLV});
+ };
+ for (unsigned I = 0, E = Data.PrivateVars.size(); I < E; ++I)
+ AddCopy(Data.PrivateVars[I], Data.PrivateCopies[I],
+ /*ElemInit=*/nullptr, Address::invalid());
+ for (unsigned I = 0, E = Data.FirstprivateVars.size(); I < E; ++I) {
+ const ValueDecl *VD = FirstprivatePtrs[I].first;
+ Address Initial(
+ CGF.Builder.CreateLoad(FirstprivatePtrs[I].second),
+ CGF.ConvertTypeForMem(VD->getType().getNonReferenceType()),
+ CGF.getContext().getDeclAlign(VD));
+ AddCopy(Data.FirstprivateVars[I], Data.FirstprivateCopies[I],
+ cast<VarDecl>(
+ cast<DeclRefExpr>(Data.FirstprivateInits[I])->getDecl()),
+ Initial);
+ }
+ for (unsigned I = 0, E = Data.LastprivateVars.size(); I < E; ++I)
+ AddCopy(Data.LastprivateVars[I], Data.LastprivateCopies[I],
+ /*ElemInit=*/nullptr, Address::invalid());
+ }
// Adjust mapping for internal locals by mapping actual memory instead of
// a pointer to this memory.
for (auto &Pair : UntiedLocalVars) {
@@ -5838,6 +5949,21 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
CGOpenMPRuntime::UntiedTaskLocalDeclsRAII LocalVarsScope(CGF,
UntiedLocalVars);
Action.Enter(CGF);
+ // Construct the body's copies, once, in an untied task's first part, and
+ // destroy them when the body ends.
+ for (const BodyCopyTy &Copy : BodyCopies) {
+ if (const Expr *Init = Copy.PrivateCopy->getAnyInitializer())
+ CGF.EmitOMPPrivateCopyInit(Init, Copy.PrivateCopy, Copy.ElemInit,
+ Copy.Dst, Copy.Src,
+ /*CapturesInfo=*/nullptr);
+ else if (Copy.ElemInit)
+ // A firstprivate of an array structured binding has no initializer:
+ // its bytes are copied, as emitPrivatesInit does for its initial value.
+ CGF.EmitAggregateAssign(Copy.Dst, Copy.Src, Copy.Dst.getType());
+ QualType Ty = Copy.Dst.getType();
+ if (QualType::DestructionKind DtorKind = Ty.isDestructedType())
+ CGF.pushDestroy(DtorKind, Copy.Dst.getAddress(), Ty);
+ }
BodyGen(CGF);
};
OpenMPDirectiveKind EKind = getEffectiveDirectiveKind(S);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 85d518f28c4f38..62e74df6240d79 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3894,6 +3894,15 @@ class CodeGenFunction : public CodeGenTypeCache {
void EmitOMPAggregateAssign(
Address DestAddr, Address SrcAddr, QualType OriginalType,
const llvm::function_ref<void(Address, Address)> CopyGen);
+ /// Initialize \p Dst, a task's private copy, with \p Init, the initializer
+ /// of the copy's declaration \p PrivateCopy. For a firstprivate,
+ /// \p ElemInit is the element the initializer reads, which is \p Src, and an
+ /// array is initialized element by element; for any other copy \p Src is
+ /// unused. The initializer is emitted under \p CapturesInfo if it is
+ /// non-null, and under the current captured-statement info otherwise.
+ void EmitOMPPrivateCopyInit(const Expr *Init, const VarDecl *PrivateCopy,
+ const VarDecl *ElemInit, LValue Dst, LValue Src,
+ CGCapturedStmtInfo *CapturesInfo);
/// Emit proper copying of data from one variable to another.
///
/// \param OriginalType Original type of the copied variables.
diff --git a/clang/test/OpenMP/taskgraph_task_body_copies_codegen.cpp b/clang/test/OpenMP/taskgraph_task_body_copies_codegen.cpp
new file mode 100644
index 00000000000000..50d01831f26a3b
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_task_body_copies_codegen.cpp
@@ -0,0 +1,107 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-linux-gnu -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// A task that may be recorded into a taskgraph is replayed from its recorded
+// task descriptor, so its body leaves the descriptor as it found it. The
+// descriptor holds only the initial values of the firstprivates, and the body
+// constructs a copy of every private list item of its own -- a firstprivate's
+// from its initial value, a private one by default construction -- uses
+// those, and destroys them when it ends. The task's destructor thunk destroys
+// only the initial values. An untied task's copies live in the descriptor
+// too, beside the initial values, so that they last from its first part to
+// its last. A task that cannot be recorded is unchanged.
+
+struct NT {
+ int v;
+ NT();
+ NT(const NT &);
+ ~NT();
+};
+
+void use(int);
+
+void run(int n) {
+ NT a, p;
+#pragma omp taskgraph
+ {
+#pragma omp task firstprivate(a, n) private(p)
+ {
+ use(a.v + p.v + n);
+ a.v = n;
+ }
+#pragma omp task untied firstprivate(a)
+ {
+ use(a.v);
+#pragma omp taskyield
+ use(a.v);
+ }
+#pragma omp task replayable(false) firstprivate(a)
+ use(a.v);
+ }
+}
+
+// p, the initial values of a and n, and the body's copies of a and n.
+// CHECK: %struct..kmp_privates.t = type { %struct.NT, %struct.NT, i32, %struct.NT, i32 }
+// The untied task's initial value of a and its copy.
+// CHECK: %struct..kmp_privates.t.{{[0-9]+}} = type { %struct.NT, %struct.NT }
+// The task that cannot be recorded has a's copy alone.
+// CHECK: %struct..kmp_privates.t.{{[0-9]+}} = type { %struct.NT }
+
+// CHECK-LABEL: define internal noundef i32 @.omp_task_entry.(
+// CHECK: call void %{{.+}}(ptr %{{.+}}, ptr [[P:%.priv.ptr.addr[^ ,]*]], ptr [[A:%.firstpriv.ptr.addr[^ ,]*]], ptr [[N:%.firstpriv.ptr.addr[^ ,]*]], ptr [[ACOPY:%.firstpriv.copy.ptr.addr[^ ,]*]], ptr [[NCOPY:%.firstpriv.copy.ptr.addr[^ ,)]*]])
+// CHECK-NEXT: [[PADDR:%.+]] = load ptr, ptr [[P]],
+// CHECK-NEXT: [[ACOPYADDR:%.+]] = load ptr, ptr [[ACOPY]],
+// CHECK-NEXT: [[NCOPYADDR:%.+]] = load ptr, ptr [[NCOPY]],
+// CHECK-NEXT: [[AADDR:%.+]] = load ptr, ptr [[A]],
+// CHECK-NEXT: [[NADDR:%.+]] = load ptr, ptr [[N]],
+// CHECK-NEXT: invoke void @_ZN2NTC1Ev(ptr {{.*}} [[PADDR]])
+// CHECK: invoke void @_ZN2NTC1ERKS_(ptr {{.*}} [[ACOPYADDR]], ptr {{.*}} [[AADDR]])
+// CHECK: [[NVAL:%.+]] = load i32, ptr [[NADDR]],
+// CHECK-NEXT: store i32 [[NVAL]], ptr [[NCOPYADDR]],
+// The body works on its copies.
+// CHECK-NEXT: load i32, ptr [[ACOPYADDR]],
+// CHECK-NEXT: load i32, ptr [[PADDR]],
+// CHECK: load i32, ptr [[NCOPYADDR]],
+// CHECK: invoke void @_Z3usei(
+// CHECK: [[NV:%.+]] = load i32, ptr [[NCOPYADDR]],
+// CHECK-NEXT: store i32 [[NV]], ptr [[ACOPYADDR]],
+// CHECK-NEXT: call void @_ZN2NTD1Ev(ptr {{.*}} [[ACOPYADDR]])
+// CHECK-NEXT: call void @_ZN2NTD1Ev(ptr {{.*}} [[PADDR]])
+// CHECK-NEXT: ret i32 0
+
+// The thunk destroys the initial value of a and nothing else.
+// CHECK-LABEL: define internal noundef i32 @.omp_task_destructor.(
+// CHECK-NOT: call void @_ZN2NTD1Ev(
+// CHECK: [[INIT:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr %{{.+}}, i32 0, i32 1
+// CHECK-NEXT: call void @_ZN2NTD1Ev(ptr {{.*}} [[INIT]])
+// CHECK-NOT: getelementptr inbounds nuw %struct..kmp_privates.t, ptr %{{.+}}, i32 0, i32 {{[0234]}}
+// CHECK: ret i32
+
+// The untied task makes its copy in its first part and destroys it in its
+// last.
+// CHECK-LABEL: define internal noundef i32 @.omp_task_entry..{{[0-9]+}}(
+// CHECK: call void %{{.+}}(ptr %{{.+}}, ptr [[UA:%.firstpriv.ptr.addr[^ ,]*]], ptr [[UACOPY:%.firstpriv.copy.ptr.addr[^ ,)]*]])
+// CHECK-NEXT: [[UACOPYADDR:%.+]] = load ptr, ptr [[UACOPY]],
+// CHECK-NEXT: [[UAADDR:%.+]] = load ptr, ptr [[UA]],
+// CHECK: switch i32 %{{.+}}, label %.untied.done.
+// CHECK: .untied.jmp.1.i:
+// CHECK-NEXT: invoke void @_ZN2NTC1ERKS_(ptr {{.*}} [[UACOPYADDR]], ptr {{.*}} [[UAADDR]])
+// CHECK: call i32 @__kmpc_omp_taskyield(
+// CHECK: .untied.jmp.3.i:
+// CHECK: call void @_ZN2NTD1Ev(ptr {{.*}} [[UACOPYADDR]])
+
+// The task that cannot be recorded uses its copy in place, and its thunk
+// destroys it.
+// CHECK-LABEL: define internal noundef i32 @.omp_task_entry..{{[0-9]+}}(
+// CHECK: call void %{{.+}}(ptr %{{.+}}, ptr [[RA:%.firstpriv.ptr.addr[^ ,)]*]])
+// CHECK-NEXT: [[RAADDR:%.+]] = load ptr, ptr [[RA]],
+// CHECK-NEXT: load i32, ptr [[RAADDR]],
+// CHECK-NOT: @_ZN2NT
+// CHECK: ret i32 0
+// CHECK-LABEL: define internal noundef i32 @.omp_task_destructor..{{[0-9]+}}(
+// CHECK: call void @_ZN2NTD1Ev(
+
+#endif
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_body_writes.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_body_writes.cpp
new file mode 100644
index 00000000000000..f226db10850af7
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_body_writes.cpp
@@ -0,0 +1,40 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A replay runs a recorded task from the task descriptor libomp kept when it
+// was recorded, so a task's body must not change what the descriptor holds:
+// a write to its copy of a firstprivate must not reach a later replay. Here
+// the firstprivate is a variable declared inside the region, which a replay
+// takes from the saved data environment (OpenMP 6.0 [14.3]), so every
+// encounter sees the value it was recorded with.
+
+#include <cstdio>
+
+static int Seen[4];
+static int Encounter;
+
+static void run() {
+#pragma omp taskgraph graph_id(1)
+ {
+ int X = 5;
+#pragma omp task firstprivate(X)
+ {
+ Seen[Encounter] = X;
+ X += 100;
+ }
+ }
+ ++Encounter;
+}
+
+int main() {
+#pragma omp parallel
+#pragma omp single
+ for (int R = 0; R < 4; ++R)
+ run();
+ std::printf("%d %d %d %d\n", Seen[0], Seen[1], Seen[2], Seen[3]);
+ return 0;
+}
+
+// CHECK: 5 5 5 5
>From a9c2f042279cc703e76c648f6438ac196ea15bc5 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 27 May 2026 10:23:14 -0500
Subject: [PATCH 11/29] [OpenMP] Initialisation and relocation of host-based
tasks in taskgraph
A taskgraph recording captures data from its calling context that is
not necessarily valid at replay time. This can be for essentially
two reasons: addresses of some data items may move if the taskgraph is
replayed at a different call-stack depth (e.g. for 'shared' data), or
the values of non-saved firstprivates may have changed, meaning cached
copies of those values taken by the previous patch need to be updated.
For host tasks, both these cases are handled via a compiler-generated
relocation helper function which takes a pointer to the recorded task
and a pointer to the taskgraph's captures (in the recording case, those
are the args from "entry(args)"):
void __omp_taskgraph_relocate.NN(kmp_task_t *task, void *outer_captures);
This has to be a generated function because only Clang knows the
relation between the taskgraph's captures and the task's data layout.
This function writes new projected addresses into the tasks 'shareds',
and updates the initial firstprivate values (for non-'saved' items) from
the taskgraph's captures. Member accesses also need special attention
to redirect from the recording-time 'this' to whichever 'this' is used
for the replay.
Firstprivate values are copy-constructed. For non-trivial types, that
raises questions about when the cached copies might be destructed.
For immediate firstprivate captures (non-'saved'), that should be as
soon as the task completes, as otherwise later on we may be dealing
with dangling pointers into the taskgraph invocation's calling frame.
For 'saved' firstprivates, we control the data location ourselves:
the destruction should happen when the taskgraph record is freed (reset).
This is done by extending the existing destructor thunk
with a 'which' parameter: this takes one of three values,
KMP_TASKGRAPH_DESTROY_ALL/REPLAY/RECORD, where ALL destroys everything
(as in the non-taskgraph case), and REPLAY/RECORD destroy data relevant to
the end of a taskgraph replay, or at taskgraph record destruction time.
This is an ABI break, so a bit in allocated in kmp_tasking_flags to
allow backward compatibility; the three-argument form is only used for
taskgraphs, and only when that bit is set.
On first recording a task, the initial copy of saved firstprivates needs
copy-constructing if it is non-trivial. That is handled by another
compiler-generated helper function, in emitTaskCloneFunction:
void .omp_taskgraph_clone.(kmp_task_t *task_dst,
const kmp_task_t *task_src, int);
This only deals with saved firstprivates: others are handled by the
above relocation helper at the start of each replay.
Tests have been added to cover new functionality.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/200408
---
clang/lib/CodeGen/CGOpenMPRuntime.cpp | 571 +++++++++++++++++-
clang/lib/CodeGen/CGOpenMPRuntime.h | 24 +
clang/lib/CodeGen/CGStmtOpenMP.cpp | 3 +
clang/lib/CodeGen/CodeGenFunction.h | 23 +
...kgraph_task_firstprivate_refresh_codegen.c | 67 ++
.../OpenMP/taskgraph_task_member_codegen.cpp | 49 ++
.../taskgraph_task_replay_copies_codegen.cpp | 114 ++++
.../taskgraph_template_handle_codegen.cpp | 36 ++
.../OpenMP/taskgraph_templates_codegen.cpp | 77 +++
openmp/runtime/src/i18n/en_US.txt | 1 +
openmp/runtime/src/kmp.h | 23 +-
openmp/runtime/src/kmp_tasking.cpp | 154 ++++-
...taskgraph_concurrent_replay_relocation.cpp | 79 +++
.../test/taskgraph/taskgraph_deps_27.cpp | 4 +-
.../taskgraph_firstprivate_later_writer.cpp | 66 ++
...skgraph_firstprivate_nontrivial_replay.cpp | 77 +++
.../taskgraph_firstprivate_refresh.cpp | 86 +++
.../taskgraph_firstprivate_replay_start.cpp | 58 ++
.../taskgraph_firstprivate_replay_writes.cpp | 62 ++
...askgraph_firstprivate_saved_nontrivial.cpp | 140 +++++
..._firstprivate_saved_nontrivial_selfref.cpp | 94 +++
.../taskgraph_firstprivate_saved_static.cpp | 100 +++
.../taskgraph_firstprivate_stack_depth.cpp | 116 ++++
.../taskgraph_firstprivate_untied_replay.cpp | 58 ++
.../taskgraph/taskgraph_member_replay.cpp | 68 +++
.../taskgraph_private_nontrivial_replay.cpp | 72 +++
...eplayable_lexical_shared_mixed_capture.cpp | 46 ++
..._lexical_shared_mixed_capture_taskloop.cpp | 45 ++
...layable_lexical_shared_nontrivial_type.cpp | 60 ++
...xical_shared_nontrivial_type_recursive.cpp | 88 +++
...red_nontrivial_type_recursive_taskloop.cpp | 89 +++
...exical_shared_nontrivial_type_taskloop.cpp | 59 ++
...raph_replayable_lexical_shared_pointer.cpp | 44 ++
...xical_shared_pointer_recursive_frameid.cpp | 76 +++
...red_pointer_recursive_frameid_taskloop.cpp | 95 +++
...ayable_lexical_shared_pointer_taskloop.cpp | 44 ++
...ph_replayable_lexical_shared_recursive.cpp | 46 ++
...able_lexical_shared_recursive_taskloop.cpp | 72 +++
...skgraph_replayable_lexical_shared_this.cpp | 34 ++
...kgraph_replayable_lexical_shared_works.cpp | 42 ++
...playable_lexical_shared_works_taskloop.cpp | 42 ++
...h_replayable_nonlexical_shared_fails_1.cpp | 52 ++
...ble_nonlexical_shared_fails_1_taskloop.cpp | 51 ++
...h_replayable_nonlexical_shared_in_task.cpp | 53 ++
...ble_nonlexical_shared_in_task_taskloop.cpp | 48 ++
...taskgraph_replayable_saved_stack_depth.cpp | 118 ++++
..._replayable_saved_stack_depth_taskloop.cpp | 130 ++++
.../taskgraph_shared_stack_depth.cpp | 104 ++++
.../taskgraph/taskgraph_untied_taskyield.cpp | 131 ++++
49 files changed, 3748 insertions(+), 43 deletions(-)
create mode 100644 clang/test/OpenMP/taskgraph_task_firstprivate_refresh_codegen.c
create mode 100644 clang/test/OpenMP/taskgraph_task_member_codegen.cpp
create mode 100644 clang/test/OpenMP/taskgraph_task_replay_copies_codegen.cpp
create mode 100644 clang/test/OpenMP/taskgraph_template_handle_codegen.cpp
create mode 100644 clang/test/OpenMP/taskgraph_templates_codegen.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_concurrent_replay_relocation.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_later_writer.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_nontrivial_replay.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_refresh.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_replay_start.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_replay_writes.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial_selfref.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_static.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_stack_depth.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_untied_replay.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_member_replay.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_private_nontrivial_replay.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_this.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_in_task.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_in_task_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_untied_taskyield.cpp
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 1ea11e6f1e3b23..065b9e47956b94 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -2288,6 +2288,349 @@ void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
Region->emitUntiedSwitch(CGF);
}
+namespace {
+/// Where a taskgraph replay takes the initial value of one of a recorded task's
+/// firstprivates from. A replay re-encounters every recorded construct
+/// (OpenMP 6.0 [14.3]), so a replayed task's firstprivate is initialized from
+/// its original list item [7.5.4] as that is when the replay starts. The
+/// recorded task holds the initial values, which its body copies (see
+/// OMPTaskDataTy::BodyConstructsPrivates).
+enum class ReplayInitKind {
+ /// Not a firstprivate's initial value: the body constructs it.
+ None,
+ /// A 'saved' list item [7.2]. It lives as long as the record.
+ Saved,
+ /// A firstprivate whose variable is in the taskgraph construct's enclosing
+ /// data environment.
+ Outer,
+ /// A variable with static storage duration.
+ Global,
+ /// A firstprivate of a non-static data member.
+ Member,
+};
+
+struct ReplayInitTy {
+ const CGOpenMPRuntime::TaskPrivateFieldTy *Field;
+ ReplayInitKind Kind;
+ /// For Outer, the field of the taskgraph's record that captures the variable.
+ const FieldDecl *OuterField = nullptr;
+};
+} // namespace
+
+/// The lvalue of the structured binding \p BD in the decomposed variable at
+/// \p DecompAddr, emitted with the decomposed variable temporarily mapped
+/// there, as emitPrivatesInit does.
+static LValue emitBindingAt(CodeGenFunction &CGF, const BindingDecl *BD,
+ Address DecompAddr) {
+ const auto *DD = cast<VarDecl>(BD->getDecomposedDecl());
+ auto It = CGF.findLocalDecl(DD);
+ bool WasMapped = It != CGF.localDeclMapEnd();
+ Address Saved = WasMapped ? It->second : Address::invalid();
+ if (WasMapped)
+ It->second = DecompAddr;
+ else
+ CGF.insertLocalDecl(DD, DecompAddr);
+ LValue Result = CGF.EmitLValue(BD->getBinding());
+ if (WasMapped)
+ CGF.findLocalDecl(DD)->second = Saved;
+ else
+ CGF.eraseLocalDecl(DD);
+ return Result;
+}
+
+/// Whether \p VD is the variable Sema makes for a non-static data member named
+/// in a clause in a member function -- `firstprivate(m)` -- whose list item is
+/// really `this->m`. Its initializer is the member access.
+static bool isThisMemberCapture(const VarDecl *VD) {
+ if (!isa<OMPCapturedExprDecl>(VD) || !VD->getInit())
+ return false;
+ const Expr *E = VD->getInit()->IgnoreParenImpCasts();
+ while (const auto *ME = dyn_cast<MemberExpr>(E))
+ E = ME->getBase()->IgnoreParenImpCasts();
+ return isa<CXXThisExpr>(E);
+}
+
+/// Decide how a replay updates the field of a task's privates record that is
+/// the copy of \p Original, named in its clause by \p OriginalRef, described
+/// by \p ElemInit and \p WorkingCopy (see PrivateHelpersTy). \p OuterCSI is
+/// the taskgraph construct's captured-statement info, which says what its
+/// enclosing data environment holds; for ReplayInitKind::Outer, the field of
+/// the taskgraph's record that captures the variable is returned in
+/// \p OuterField.
+static ReplayInitKind
+classifyReplayInit(const Expr *OriginalRef, const VarDecl *Original,
+ const VarDecl *ElemInit, bool WorkingCopy,
+ const OMPTaskDataTy &Data,
+ const CodeGenFunction::CGCapturedStmtInfo *OuterCSI,
+ const FieldDecl **OuterField = nullptr) {
+ if (!ElemInit || WorkingCopy)
+ return ReplayInitKind::None; // Not a firstprivate's initial value.
+ // `saved` is per list item, which may be one binding of a structured
+ // binding declaration rather than Original, the variable it decomposes.
+ const ValueDecl *ClauseDecl =
+ OriginalRef ? cast<DeclRefExpr>(OriginalRef)->getDecl() : Original;
+ if (Data.SavedFirstprivates.contains(
+ cast<ValueDecl>(ClauseDecl->getCanonicalDecl())))
+ return ReplayInitKind::Saved;
+ if (Original->hasGlobalStorage())
+ return ReplayInitKind::Global;
+ if (isThisMemberCapture(Original) && OuterCSI && OuterCSI->getThisFieldDecl())
+ return ReplayInitKind::Member;
+ if (const FieldDecl *Outer =
+ OuterCSI ? OuterCSI->lookup(Original) : nullptr) {
+ if (OuterField)
+ *OuterField = Outer;
+ return ReplayInitKind::Outer;
+ }
+ return ReplayInitKind::Saved;
+}
+
+/// Decide how a replay updates each field in \p Fields (see
+/// classifyReplayInit).
+static SmallVector<ReplayInitTy, 4>
+classifyReplayInits(const OMPTaskDataTy &Data,
+ const CodeGenFunction::CGCapturedStmtInfo *OuterCSI,
+ ArrayRef<CGOpenMPRuntime::TaskPrivateFieldTy> Fields) {
+ SmallVector<ReplayInitTy, 4> Inits;
+ for (const auto &F : Fields) {
+ ReplayInitTy Init{&F, ReplayInitKind::None};
+ Init.Kind =
+ classifyReplayInit(F.OriginalRef, F.Original, F.PrivateElemInit,
+ F.WorkingCopy, Data, OuterCSI, &Init.OuterField);
+ Inits.push_back(Init);
+ }
+ return Inits;
+}
+
+/// The record of type \p RD at the address in local variable \p Arg.
+static LValue loadRecordArg(CodeGenFunction &CGF, const ImplicitParamDecl *Arg,
+ const RecordDecl *RD) {
+ llvm::Value *Raw = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Arg));
+ return CGF.MakeNaturalAlignRawAddrLValue(
+ Raw, CGF.getContext().getCanonicalTagType(RD));
+}
+
+/// Emit a helper with the runtime relocation signature (kmp_task_relocate_t):
+/// void relocate(kmp_task_t *task, void *outer_captures);
+///
+/// At the start of every replay the runtime invokes this helper to make the
+/// recorded task descriptor simulate what a task this replay's encounter of the
+/// construct would have generated. It does two things.
+///
+/// * It refreshes the task's shared-pointer table. Each capture (a
+/// shared-by-ref variable or \c this) that the task body actually
+/// dereferences at execution time is re-projected from the freshly
+/// reconstructed outer record passed as \p outer_captures and stored back
+/// into \c task->shareds. Captures that the body cannot observe a changed
+/// address for across replays are skipped here.
+///
+/// * It copy-constructs the initial value of each firstprivate whose variable
+/// is in the taskgraph construct's enclosing data environment from the
+/// variable (see ReplayInitKind). Saved firstprivates are skipped.
+///
+/// When there is nothing to do, the helper is still emitted, as a non-null
+/// no-op: the runtime only inspects null-vs-non-null, and null means that the
+/// task cannot be relocated (e.g. it refers to shareds which cannot be found in
+/// OuterCSI).
+static llvm::Function *emitTaskRelocationFunction(
+ CodeGenModule &CGM, SourceLocation Loc, const CapturedStmt &CS,
+ const CodeGenFunction::CGCapturedStmtInfo *OuterCSI,
+ const OMPTaskDataTy &Data, const RecordDecl *TaskWithPrivatesRD,
+ ArrayRef<ReplayInitTy> Inits) {
+ ASTContext &C = CGM.getContext();
+
+ // Variables that don't need their shareds slot refreshed across replays
+ // because the body sources them from the per-task '.kmp_privates.t'
+ // copy: the firstprivate list items.
+ llvm::SmallPtrSet<const VarDecl *, 8> NoRelocateFirstprivateVars;
+ for (const Expr *E : Data.FirstprivateVars) {
+ if (!E)
+ continue;
+ if (const auto *DRE = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts()))
+ if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl()))
+ NoRelocateFirstprivateVars.insert(VD->getCanonicalDecl());
+ }
+
+ // A capture is "no-op-safe" with respect to taskgraph replay when
+ // refreshing its shareds slot is provably unnecessary - either because
+ // the body never reads from that slot (firstprivate) or because the
+ // captured pointer is a link-time-fixed address and is therefore
+ // identical at every replay (static storage duration).
+ auto IsNoOpRelocate = [&](const CapturedStmt::Capture &Cap) {
+ if (Cap.capturesThis() || !Cap.capturesVariable())
+ return false;
+ const VarDecl *VD = Cap.getCapturedVar();
+ if (VD->hasGlobalStorage())
+ return true;
+ return NoRelocateFirstprivateVars.contains(VD->getCanonicalDecl());
+ };
+
+ auto LookupOuterField =
+ [&](const CapturedStmt::Capture &Cap) -> const FieldDecl * {
+ if (!OuterCSI)
+ return nullptr;
+ return Cap.capturesThis() ? OuterCSI->getThisFieldDecl()
+ : OuterCSI->lookup(Cap.getCapturedVar());
+ };
+
+ // Bail out before emitting any IR if a genuinely-shared capture cannot be
+ // resolved in the containing context. No-op-safe captures don't participate
+ // in this check; they simply cause the helper to skip their slot below.
+ if (llvm::any_of(CS.captures(), [&](const CapturedStmt::Capture &Cap) {
+ assert((Cap.capturesThis() || Cap.capturesVariable()) &&
+ "OpenMP task capture must be shared-by-ref or 'this'");
+ return !IsNoOpRelocate(Cap) && !LookupOuterField(Cap);
+ }))
+ return nullptr;
+
+ // void relocate(void *task, void *outer_captures)
+ auto *TaskArg =
+ ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
+ C.VoidPtrTy, ImplicitParamKind::Other);
+ auto *OuterArg =
+ ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
+ C.VoidPtrTy, ImplicitParamKind::Other);
+ FunctionArgList Args{TaskArg, OuterArg};
+ const CGFunctionInfo &FnInfo =
+ CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
+
+ std::string Name =
+ CGM.getOpenMPRuntime().getName({"omp", "taskgraph", "relocate", ""});
+ auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(FnInfo),
+ llvm::GlobalValue::InternalLinkage, Name,
+ &CGM.getModule());
+ CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
+ if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
+ Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
+ Fn->setDoesNotRecurse();
+
+ CodeGenFunction CGF(CGM);
+ CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
+
+ CGBuilderTy &Bld = CGF.Builder;
+ CharUnits PtrAlign = CGF.getPointerAlign();
+
+ // Base of the reconstructed outer record for this replay.
+ llvm::Value *OuterRaw = Bld.CreateLoad(CGF.GetAddrOfLocalVar(OuterArg));
+ auto OuterBaseFor = [&](const FieldDecl *Field) {
+ QualType OuterTy =
+ C.getCanonicalTagType(cast<RecordDecl>(Field->getDeclContext()));
+ return CGF.MakeAddrLValue(
+ Address(OuterRaw, CGF.ConvertTypeForMem(OuterTy), PtrAlign), OuterTy);
+ };
+ // The replay's `this`. The record holds the pointer itself, not a
+ // reference to it.
+ auto LoadOuterThis = [&]() {
+ const FieldDecl *ThisField = OuterCSI->getThisFieldDecl();
+ return CGF.EmitLoadOfScalar(
+ CGF.EmitLValueForField(OuterBaseFor(ThisField), ThisField), Loc);
+ };
+
+ // kmp_task_t::shareds is the first field of the runtime task descriptor;
+ // load it to obtain the void* shared table that we will refresh in place.
+ // The table holds one void* per by-ref capture.
+ llvm::Value *TaskRaw = Bld.CreateLoad(CGF.GetAddrOfLocalVar(TaskArg));
+ llvm::Value *SharedRaw =
+ Bld.CreateLoad(Address(TaskRaw, CGF.VoidPtrTy, PtrAlign));
+ Address SharedTable(SharedRaw, CGF.VoidPtrTy, PtrAlign);
+
+ unsigned Index = 0;
+ for (const CapturedStmt::Capture &Cap : CS.captures()) {
+ // Always advance the slot index so that we stay aligned with the
+ // shareds-table layout established at task allocation.
+ unsigned ThisIndex = Index++;
+ if (IsNoOpRelocate(Cap))
+ continue;
+ // Project the capture's referent from the freshly reconstructed outer
+ // record. EmitLValueForField auto-loads the outer reference field, so
+ // the resulting pointer is the live referent address (not the slot).
+ // `this` is held by value, so it is the slot's contents.
+ const FieldDecl *OuterField = LookupOuterField(Cap);
+ assert(OuterField && "capture should already have been rejected");
+ llvm::Value *Mapped =
+ Cap.capturesThis()
+ ? LoadOuterThis()
+ : CGF.EmitLValueForField(OuterBaseFor(OuterField), OuterField)
+ .getPointer(CGF);
+ Mapped = Bld.CreatePointerBitCastOrAddrSpaceCast(Mapped, CGM.VoidPtrTy);
+ Bld.CreateStore(Mapped, Bld.CreateConstGEP(SharedTable, ThisIndex));
+ }
+
+ if (llvm::any_of(Inits, [](const ReplayInitTy &I) {
+ return I.Kind != ReplayInitKind::None &&
+ I.Kind != ReplayInitKind::Saved;
+ })) {
+ auto PrivatesFI = std::next(TaskWithPrivatesRD->field_begin());
+ LValue PrivatesBase = CGF.EmitLValueForField(
+ loadRecordArg(CGF, TaskArg, TaskWithPrivatesRD), *PrivatesFI);
+ CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
+ for (const ReplayInitTy &I : Inits) {
+ const auto &F = *I.Field;
+ LValue Src;
+ switch (I.Kind) {
+ case ReplayInitKind::None:
+ case ReplayInitKind::Saved:
+ continue;
+ case ReplayInitKind::Outer:
+ Src = CGF.EmitLValueForField(OuterBaseFor(I.OuterField), I.OuterField);
+ break;
+ case ReplayInitKind::Member: {
+ // Evaluate the member access the clause named against the replay's
+ // `this`.
+ const CXXRecordDecl *ThisRD =
+ OuterCSI->getThisFieldDecl()->getType()->getPointeeCXXRecordDecl();
+ CodeGenFunction::CXXThisOverrideScope ThisScope(
+ CGF, LoadOuterThis(), CGM.getClassPointerAlignment(ThisRD));
+ Src = CGF.EmitLValue(F.Original->getInit());
+ break;
+ }
+ case ReplayInitKind::Global: {
+ DeclRefExpr DRE(C, const_cast<VarDecl *>(F.Original),
+ /*RefersToEnclosingVariableOrCapture=*/false,
+ F.Original->getType().getNonReferenceType(), VK_LValue,
+ Loc);
+ Src = CGF.EmitLValue(&DRE);
+ break;
+ }
+ }
+ // A structured binding's initial value is its subobject of the
+ // decomposed variable Src names.
+ if (const auto *DRE = dyn_cast_or_null<DeclRefExpr>(F.OriginalRef))
+ if (const auto *BD = dyn_cast<BindingDecl>(DRE->getDecl()))
+ Src = emitBindingAt(CGF, BD, Src.getAddress());
+ LValue Dst = CGF.EmitLValueForField(PrivatesBase, F.Field);
+ if (const Expr *Init = F.PrivateCopy->getAnyInitializer())
+ CGF.EmitOMPPrivateCopyInit(Init, F.PrivateCopy, F.PrivateElemInit, Dst,
+ Src, &CapturesInfo);
+ else
+ // An array binding's copy has no initializer; its bytes are copied.
+ CGF.EmitAggregateAssign(Dst, Src, Dst.getType());
+ }
+ }
+
+ CGF.FinishFunction();
+ return Fn;
+}
+
+/// Emit the helper a recorded task needs to be replayed, its relocation
+/// function, and return it as an argument of __kmpc_taskgraph_task or
+/// __kmpc_taskgraph_taskloop: null when there is none.
+static llvm::Value *emitTaskReplayHelper(
+ CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
+ const OMPTaskDataTy &Data, const RecordDecl *TaskWithPrivatesRD,
+ ArrayRef<CGOpenMPRuntime::TaskPrivateFieldTy> PrivateFields) {
+ assert(Data.BodyConstructsPrivates &&
+ "a recorded task's body must construct its own private copies");
+ const auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
+ SmallVector<ReplayInitTy, 4> Inits =
+ classifyReplayInits(Data, CGF.CapturedStmtInfo, PrivateFields);
+ llvm::Function *Fn = emitTaskRelocationFunction(
+ CGF.CGM, Loc, *CS, CGF.CapturedStmtInfo, Data, TaskWithPrivatesRD, Inits);
+ if (!Fn)
+ return llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Fn, CGF.CGM.VoidPtrTy);
+}
+
void CGOpenMPRuntime::emitTaskgraphCall(CodeGenFunction &CGF,
SourceLocation Loc,
const OMPExecutableDirective &D,
@@ -3463,10 +3806,29 @@ emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc,
return TaskEntry;
}
+/// Emit the destructors thunk of a task, which destroys the task's private
+/// copies that are constructed with its descriptor (see isDescriptorCopy):
+/// \code
+/// kmp_int32 .omp_task_destructor.(kmp_int32 gtid, kmp_task_t *task);
+/// \endcode
+/// A task that may be recorded into a taskgraph holds the initial values of its
+/// firstprivates in its descriptor, and the record keeps them for different
+/// lengths of time depending on where a replay takes them from (see
+/// ReplayInitKind). So its thunk takes a third argument, which of them to
+/// destroy, and the task carries a flag saying so:
+/// \code
+/// kmp_int32 .omp_task_destructor.(kmp_int32 gtid, kmp_task_t *task,
+/// kmp_int32 which);
+/// \endcode
+/// where \c which is 0 for all of them (a task that runs outside a record), 1
+/// for those a replay reads afresh (when a replay ends, or a taskloop's task is
+/// recorded) and 2 for the saved data environment (when the record is
+/// freed). \p OuterCSI is the taskgraph construct's captured-statement info.
static llvm::Value *emitDestructorsFunction(
CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty,
QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy,
- const OMPTaskDataTy &Data, ArrayRef<PrivateDataTy> Privates) {
+ const OMPTaskDataTy &Data, ArrayRef<PrivateDataTy> Privates,
+ const CodeGenFunction::CGCapturedStmtInfo *OuterCSI) {
ASTContext &C = CGM.getContext();
auto *GtidArg =
ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
@@ -3475,6 +3837,13 @@ static llvm::Value *emitDestructorsFunction(
C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
KmpTaskTWithPrivatesPtrQTy.withRestrict(), ImplicitParamKind::Other);
FunctionArgList Args{GtidArg, TaskTypeArg};
+ ImplicitParamDecl *WhichArg = nullptr;
+ if (Data.BodyConstructsPrivates) {
+ WhichArg = ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc,
+ /*Id=*/nullptr, KmpInt32Ty,
+ ImplicitParamKind::Other);
+ Args.push_back(WhichArg);
+ }
const auto &DestructorFnInfo =
CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args);
llvm::FunctionType *DestructorFnTy =
@@ -3500,18 +3869,57 @@ static llvm::Value *emitDestructorsFunction(
KmpTaskTWithPrivatesQTy->castAsRecordDecl();
auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
Base = CGF.EmitLValueForField(Base, *FI);
- auto PI = Privates.begin();
- for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
- const PrivateHelpersTy &P = (PI++)->second;
- // The body destroys the copies it constructs.
- if (Data.BodyConstructsPrivates && !isDescriptorCopy(P, Data))
- continue;
- if (QualType::DestructionKind DtorKind =
- Field->getType().isDestructedType()) {
- LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
- CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
+ if (!WhichArg) {
+ for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
+ if (QualType::DestructionKind DtorKind =
+ Field->getType().isDestructedType()) {
+ LValue FieldLValue = CGF.EmitLValueForField(Base, Field);
+ CGF.pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
+ }
}
- }
+ CGF.FinishFunction();
+ return DestructorFn;
+ }
+
+ // The values of \c which, as in kmp.h.
+ enum { DestroyAll = 0, DestroyReplay = 1, DestroyRecord = 2 };
+ llvm::BasicBlock *AllBB = CGF.createBasicBlock("omp.destroy.all");
+ llvm::BasicBlock *ReplayBB = CGF.createBasicBlock("omp.destroy.replay");
+ llvm::BasicBlock *RecordBB = CGF.createBasicBlock("omp.destroy.record");
+ llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.destroy.done");
+ llvm::SwitchInst *WhichSwitch = CGF.Builder.CreateSwitch(
+ CGF.EmitLoadOfScalar(CGF.GetAddrOfLocalVar(WhichArg), /*Volatile=*/false,
+ KmpInt32Ty, Loc),
+ DoneBB, 3);
+ WhichSwitch->addCase(CGF.Builder.getInt32(DestroyAll), AllBB);
+ WhichSwitch->addCase(CGF.Builder.getInt32(DestroyReplay), ReplayBB);
+ WhichSwitch->addCase(CGF.Builder.getInt32(DestroyRecord), RecordBB);
+ // Destroy the initial values \p Wanted says to.
+ auto EmitDestroys = [&](llvm::BasicBlock *BB,
+ llvm::function_ref<bool(ReplayInitKind)> Wanted) {
+ CGF.EmitBlock(BB);
+ auto PI = Privates.begin();
+ for (const auto *Field : FI->getType()->castAsRecordDecl()->fields()) {
+ const PrivateHelpersTy &P = (PI++)->second;
+ if (!isDescriptorCopy(P, Data))
+ continue;
+ QualType::DestructionKind DtorKind = Field->getType().isDestructedType();
+ if (!DtorKind || !Wanted(classifyReplayInit(
+ P.OriginalRef, P.Original, P.PrivateElemInit,
+ P.WorkingCopy, Data, OuterCSI)))
+ continue;
+ CGF.emitDestroy(CGF.EmitLValueForField(Base, Field).getAddress(),
+ Field->getType(), CGF.getDestroyer(DtorKind),
+ /*useEHCleanupForArray=*/false);
+ }
+ CGF.EmitBranch(DoneBB);
+ };
+ EmitDestroys(AllBB, [](ReplayInitKind) { return true; });
+ EmitDestroys(ReplayBB,
+ [](ReplayInitKind K) { return K != ReplayInitKind::Saved; });
+ EmitDestroys(RecordBB,
+ [](ReplayInitKind K) { return K == ReplayInitKind::Saved; });
+ CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
CGF.FinishFunction();
return DestructorFn;
}
@@ -3964,6 +4372,99 @@ emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
return TaskDup;
}
+/// Whether the copy libomp keeps of a recorded task needs \p P
+/// copy-constructed from the task the recording runs: a firstprivate's initial
+/// value that belongs to the saved data environment (ReplayInitKind::Saved),
+/// whose initialization is not trivial. The task's relocation helper
+/// constructs the other initial values afresh for every replay, so the copy
+/// needs nothing of them but storage. \p OuterCSI is the taskgraph
+/// construct's captured-statement info.
+static bool
+isClonedInitialValue(CodeGenFunction &CGF, const PrivateHelpersTy &P,
+ const OMPTaskDataTy &Data,
+ const CodeGenFunction::CGCapturedStmtInfo *OuterCSI) {
+ if (!isDescriptorCopy(P, Data) ||
+ classifyReplayInit(P.OriginalRef, P.Original, P.PrivateElemInit,
+ P.WorkingCopy, Data,
+ OuterCSI) != ReplayInitKind::Saved)
+ return false;
+ const Expr *Init = P.PrivateCopy->getAnyInitializer();
+ return isa_and_nonnull<CXXConstructExpr>(Init) &&
+ !CGF.isTrivialInitializer(Init);
+}
+
+/// Emit the function that completes the copy libomp keeps of a task the
+/// recording runs:
+/// \code
+/// void .omp_taskgraph_clone.(kmp_task_t *task_dst, const kmp_task_t *task_src,
+/// int);
+/// \endcode
+/// libomp copies the descriptor bitwise, which leaves a broken copy of an
+/// initial value whose type is not trivially copyable, so this copy-constructs
+/// those of the saved data environment from the source (see
+/// isClonedInitialValue). The rest are left as bytes, which the relocation
+/// helper constructs over when a replay starts.
+///
+/// See also emitTaskDupFunction (used for taskloop tasks).
+static llvm::Value *emitTaskCloneFunction(
+ CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D,
+ QualType KmpTaskTWithPrivatesPtrQTy,
+ const RecordDecl *KmpTaskTWithPrivatesQTyRD, const OMPTaskDataTy &Data,
+ ArrayRef<PrivateDataTy> Privates,
+ const CodeGenFunction::CGCapturedStmtInfo *OuterCSI) {
+ ASTContext &C = CGM.getContext();
+ auto *DstArg = ImplicitParamDecl::Create(
+ C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy,
+ ImplicitParamKind::Other);
+ auto *SrcArg = ImplicitParamDecl::Create(
+ C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy,
+ ImplicitParamKind::Other);
+ auto *LastprivArg =
+ ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
+ ImplicitParamKind::Other);
+ FunctionArgList Args{DstArg, SrcArg, LastprivArg};
+ const auto &FnInfo =
+ CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
+ std::string Name =
+ CGM.getOpenMPRuntime().getName({"omp_taskgraph_clone", ""});
+ auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(FnInfo),
+ llvm::GlobalValue::InternalLinkage, Name,
+ &CGM.getModule());
+ CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
+ if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
+ Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
+ Fn->setDoesNotRecurse();
+ CodeGenFunction CGF(CGM);
+ CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
+
+ auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
+ auto PrivatesOf = [&](const ImplicitParamDecl *Arg) {
+ return CGF.EmitLValueForField(
+ CGF.EmitLoadOfPointerLValue(
+ CGF.GetAddrOfLocalVar(Arg),
+ KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>()),
+ *PrivatesFI);
+ };
+ LValue DstBase = PrivatesOf(DstArg);
+ LValue SrcBase = PrivatesOf(SrcArg);
+ CodeGenFunction::CGCapturedStmtInfo CapturesInfo(
+ *D.getCapturedStmt(OMPD_task));
+ auto FI = PrivatesFI->getType()->castAsRecordDecl()->field_begin();
+ for (const PrivateDataTy &Pair : Privates) {
+ const FieldDecl *Field = *FI++;
+ if (!isClonedInitialValue(CGF, Pair.second, Data, OuterCSI))
+ continue;
+ const VarDecl *VD = Pair.second.PrivateCopy;
+ const Expr *Init = VD->getAnyInitializer();
+ CGF.EmitOMPPrivateCopyInit(Init, VD, Pair.second.PrivateElemInit,
+ CGF.EmitLValueForField(DstBase, Field),
+ CGF.EmitLValueForField(SrcBase, Field),
+ &CapturesInfo);
+ }
+ CGF.FinishFunction();
+ return Fn;
+}
+
/// Checks if destructor function is required to be generated.
/// \return true if cleanups are required, false otherwise.
static bool checkDestructorsRequired(ArrayRef<PrivateDataTy> Privates,
@@ -4238,6 +4739,7 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
DetachableFlag = 0x40,
FreeAgentFlag = 0x80,
TransparentFlag = 0x100,
+ TaskgraphDestructorsFlag = 0x200,
};
unsigned Flags = Data.Tied ? TiedFlag : 0;
bool NeedsCleanup = false;
@@ -4245,6 +4747,10 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
NeedsCleanup = checkDestructorsRequired(Privates, Data);
if (NeedsCleanup)
Flags = Flags | DestructorsFlag;
+ // The thunk of a task that may be recorded takes which of its initial
+ // values to destroy (see emitDestructorsFunction).
+ if (NeedsCleanup && Data.BodyConstructsPrivates)
+ Flags = Flags | TaskgraphDestructorsFlag;
}
if (const auto *Clause = D.getSingleClause<OMPThreadsetClause>()) {
OpenMPThreadsetKind Kind = Clause->getThreadsetKind();
@@ -4478,7 +4984,20 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
}
// Emit initial values for private copies (if any).
TaskResultTy Result;
+ Result.KmpTaskTWithPrivatesQTyRD = KmpTaskTWithPrivatesQTyRD;
if (!Privates.empty()) {
+ // The privates record has one field per entry of Privates, in order.
+ const auto *PrivatesRD = std::next(KmpTaskTWithPrivatesQTyRD->field_begin())
+ ->getType()
+ ->castAsRecordDecl();
+ auto PrivateField = PrivatesRD->field_begin();
+ for (const PrivateDataTy &Pair : Privates) {
+ Result.PrivateFields.push_back(
+ {Pair.second.OriginalRef, Pair.second.Original,
+ Pair.second.PrivateCopy, Pair.second.PrivateElemInit, *PrivateField,
+ Pair.second.WorkingCopy});
+ ++PrivateField;
+ }
emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
SharedsTy, SharedsPtrTy, Data, Privates,
/*ForDup=*/false);
@@ -4489,6 +5008,17 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
/*WithLastIter=*/!Data.LastprivateVars.empty());
+ } else if (ForTaskgraph && Data.BodyConstructsPrivates &&
+ llvm::any_of(Privates, [&](const PrivateDataTy &Pair) {
+ return isClonedInitialValue(CGF, Pair.second, Data,
+ CGF.CapturedStmtInfo);
+ })) {
+ // A recorded task is kept as a copy of the task the recording runs, and
+ // the saved initial values in it have to be constructed, not copied
+ // bitwise.
+ Result.TaskDupFn = emitTaskCloneFunction(
+ CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
+ Data, Privates, CGF.CapturedStmtInfo);
}
}
// Fields of union "kmp_cmplrdata_t" for destructors and priority.
@@ -4500,7 +5030,7 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
if (NeedsCleanup) {
llvm::Value *DestructorFn = emitDestructorsFunction(
CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
- KmpTaskTWithPrivatesQTy, Data, Privates);
+ KmpTaskTWithPrivatesQTy, Data, Privates, CGF.CapturedStmtInfo);
LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI);
LValue DestructorsLV = CGF.EmitLValueForField(
Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
@@ -5066,9 +5596,13 @@ void CGOpenMPRuntime::emitTaskCall(
TGTaskArgs[6] = NumOfElements;
TGTaskArgs[7] = DependenciesArray.emitRawPointer(CGF);
}
- // The task's relocation and clone helpers: none yet.
- TGTaskArgs[8] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
- TGTaskArgs[9] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ TGTaskArgs[8] = emitTaskReplayHelper(CGF, Loc, D, Data,
+ Result.KmpTaskTWithPrivatesQTyRD,
+ Result.PrivateFields);
+ TGTaskArgs[9] = Result.TaskDupFn
+ ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+ Result.TaskDupFn, CGF.VoidPtrTy)
+ : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), OMPRTL___kmpc_taskgraph_task),
TGTaskArgs);
@@ -5295,8 +5829,9 @@ void CGOpenMPRuntime::emitTaskLoopCall(
? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
TaskInitResult.TaskDupFn, CGF.VoidPtrTy)
: llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
- // The task's relocation helper: none yet.
- TGTaskLoopArgs[13] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ TGTaskLoopArgs[13] = emitTaskReplayHelper(
+ CGF, Loc, D, Data, TaskInitResult.KmpTaskTWithPrivatesQTyRD,
+ TaskInitResult.PrivateFields);
CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
CGM.getModule(), OMPRTL___kmpc_taskgraph_taskloop),
TGTaskLoopArgs);
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 01e520f7f40d7f..7e1f9bdfc687d1 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -96,6 +96,10 @@ struct OMPTaskDataTy final {
SmallVector<const Expr *, 4> FirstprivateVars;
SmallVector<const Expr *, 4> FirstprivateCopies;
SmallVector<const Expr *, 4> FirstprivateInits;
+ /// The firstprivate list items whose clause carried the `saved` modifier,
+ /// which a taskgraph replay initialises from the value recorded with the
+ /// task rather than from the variable as it stands.
+ llvm::SmallPtrSet<const ValueDecl *, 4> SavedFirstprivates;
SmallVector<const Expr *, 4> LastprivateVars;
SmallVector<const Expr *, 4> LastprivateCopies;
SmallVector<const Expr *, 4> ReductionVars;
@@ -236,6 +240,21 @@ class ReductionCodeGen {
class CGOpenMPRuntime {
public:
+ /// One field of a task's privates record: the list item it is a copy of, the
+ /// copy's declaration (whose initializer constructs it) and, for a
+ /// firstprivate, the element the initializer reads. A private local of an
+ /// untied task has only Original. When the body constructs its own copies
+ /// (OMPTaskDataTy::BodyConstructsPrivates), a firstprivate has two fields:
+ /// its initial value, and the body's copy, which is the WorkingCopy.
+ struct TaskPrivateFieldTy {
+ const Expr *OriginalRef = nullptr;
+ const VarDecl *Original = nullptr;
+ const VarDecl *PrivateCopy = nullptr;
+ const VarDecl *PrivateElemInit = nullptr;
+ const FieldDecl *Field = nullptr;
+ bool WorkingCopy = false;
+ };
+
/// Allows to disable automatic handling of functions used in target regions
/// as those marked as `omp declare target`.
class DisableAutoDeclareTargetRAII {
@@ -563,6 +582,11 @@ class CGOpenMPRuntime {
LValue TDBase;
const RecordDecl *KmpTaskTQTyRD = nullptr;
llvm::Value *TaskDupFn = nullptr;
+ /// The task descriptor with its privates, and every field of its privates
+ /// record, so that a taskgraph replay can re-read the initial values of
+ /// the task's firstprivates (see emitTaskRelocationFunction).
+ const RecordDecl *KmpTaskTWithPrivatesQTyRD = nullptr;
+ SmallVector<TaskPrivateFieldTy, 4> PrivateFields;
};
/// Emit task region for the task directive. The task region is emitted in
/// several steps:
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 62fe474624be88..3babe21a23dac5 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -5537,6 +5537,9 @@ void CodeGenFunction::EmitOMPTaskBasedDirective(
Data.FirstprivateVars.push_back(*IRef);
Data.FirstprivateCopies.push_back(IInit);
Data.FirstprivateInits.push_back(*IElemInitRef);
+ if (C->getKind() == OMPC_FIRSTPRIVATE_saved)
+ Data.SavedFirstprivates.insert(
+ cast<ValueDecl>(OrigDecl->getCanonicalDecl()));
}
++IRef;
++IElemInitRef;
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 62e74df6240d79..23ddc8befed01c 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -1907,6 +1907,29 @@ class CodeGenFunction : public CodeGenTypeCache {
Address OldCXXDefaultInitExprThis;
};
+ /// Within this scope, the value of 'this' is overridden to be \p This. A
+ /// helper emitted outside the member function an expression comes from uses
+ /// it to evaluate the expression against an object it has found itself.
+ class CXXThisOverrideScope {
+ public:
+ CXXThisOverrideScope(CodeGenFunction &CGF, llvm::Value *This,
+ CharUnits Alignment)
+ : CGF(CGF), OldCXXThisValue(CGF.CXXThisValue),
+ OldCXXThisAlignment(CGF.CXXThisAlignment) {
+ CGF.CXXThisValue = This;
+ CGF.CXXThisAlignment = Alignment;
+ }
+ ~CXXThisOverrideScope() {
+ CGF.CXXThisValue = OldCXXThisValue;
+ CGF.CXXThisAlignment = OldCXXThisAlignment;
+ }
+
+ private:
+ CodeGenFunction &CGF;
+ llvm::Value *OldCXXThisValue;
+ CharUnits OldCXXThisAlignment;
+ };
+
/// The scope of a CXXDefaultInitExpr. Within this scope, the value of 'this'
/// is overridden to be the object under construction.
class CXXDefaultInitExprScope {
diff --git a/clang/test/OpenMP/taskgraph_task_firstprivate_refresh_codegen.c b/clang/test/OpenMP/taskgraph_task_firstprivate_refresh_codegen.c
new file mode 100644
index 00000000000000..a523e39bbc8980
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_task_firstprivate_refresh_codegen.c
@@ -0,0 +1,67 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c -triple x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// A replay generates its tasks afresh (OpenMP 6.0 [14.3]), and the body of a
+// task that may be recorded constructs its private copies from the initial
+// values its task descriptor holds. libomp reuses the recorded descriptor, so
+// the task's relocation helper reads those initial values again at the start
+// of each replay. A replay starts from the taskgraph construct's enclosing
+// data environment, so a firstprivate of a variable that environment has takes
+// the value the variable has at replay: through the taskgraph's record for an
+// automatic variable, and from its fixed address for one with static storage
+// duration.
+//
+// The saved data environment supplies the rest: a list item with the `saved`
+// modifier, and a variable declared inside the region, which has no value at
+// replay. Their initial values stay as the task was recorded with them.
+
+struct Pt {
+ int X, Y;
+};
+int G;
+int Out;
+
+void run(int n, struct Pt p, int s) {
+#pragma omp taskgraph
+ {
+ int in = n;
+#pragma omp task firstprivate(n, p, G, in) firstprivate(saved : s) shared(Out)
+ Out = n + p.Y + G + in + s;
+ }
+}
+
+// The initial values of n, p, G, in and s, in that order, then the body's
+// copies of them.
+// CHECK: %struct..kmp_privates.t = type { i32, %struct.Pt, i32, i32, i32, i32, %struct.Pt, i32, i32, i32 }
+
+// CHECK: call i32 @__kmpc_taskgraph_task({{.*}}, ptr @.omp.taskgraph.relocate., ptr null)
+
+// The helper reads n, p and G; in and s keep the values they were recorded
+// with.
+// CHECK: define internal void @.omp.taskgraph.relocate.(ptr noundef %0, ptr noundef %1)
+// CHECK: [[REC:%.+]] = load ptr, ptr %.addr1,
+// CHECK-NEXT: [[TASK:%.+]] = load ptr, ptr %.addr,
+// CHECK-NEXT: load ptr, ptr [[TASK]],
+// CHECK-NEXT: [[TASK2:%.+]] = load ptr, ptr %.addr,
+// CHECK-NEXT: [[PRIV:%.+]] = getelementptr inbounds nuw %struct.kmp_task_t_with_privates, ptr [[TASK2]], i32 0, i32 1
+// n: a scalar, through the record.
+// CHECK-NEXT: [[NF:%.+]] = getelementptr inbounds nuw %struct.anon, ptr [[REC]], i32 0, i32 0
+// CHECK-NEXT: [[NADDR:%.+]] = load ptr, ptr [[NF]],
+// CHECK-NEXT: [[NDST:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[PRIV]], i32 0, i32 0
+// CHECK-NEXT: [[NVAL:%.+]] = load i32, ptr [[NADDR]],
+// CHECK-NEXT: store i32 [[NVAL]], ptr [[NDST]],
+// p: an aggregate, through the record.
+// CHECK-NEXT: [[PF:%.+]] = getelementptr inbounds nuw %struct.anon, ptr [[REC]], i32 0, i32 1
+// CHECK-NEXT: [[PADDR:%.+]] = load ptr, ptr [[PF]],
+// CHECK-NEXT: [[PDST:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[PRIV]], i32 0, i32 1
+// CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[PDST]], ptr align 4 [[PADDR]], i64 8, i1 false)
+// G: from its own address.
+// CHECK-NEXT: [[GDST:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[PRIV]], i32 0, i32 2
+// CHECK-NEXT: [[GVAL:%.+]] = load i32, ptr @G,
+// CHECK-NEXT: store i32 [[GVAL]], ptr [[GDST]],
+// CHECK-NEXT: ret void
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_task_member_codegen.cpp b/clang/test/OpenMP/taskgraph_task_member_codegen.cpp
new file mode 100644
index 00000000000000..86af72d8169395
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_task_member_codegen.cpp
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// A taskgraph in a member function. A replay starts from the taskgraph
+// construct's enclosing data environment (OpenMP 6.0 [14.3]), whose `this` is
+// the object the member function was called for this time, so when a replay
+// starts the task's relocation helper takes `this` from the taskgraph's
+// reconstructed record:
+//
+// - the task's own capture of `this` is set to it;
+// - the initial value of a firstprivate of a non-static data member, whose
+// list item is `this->m`, is read from that member of it.
+
+struct S {
+ int m;
+ int out;
+ void run() {
+#pragma omp taskgraph
+ {
+#pragma omp task firstprivate(m)
+ out = m;
+ }
+ }
+};
+
+void f(S &s) { s.run(); }
+
+// The record holds `this` in its first field, by value.
+// CHECK: define internal void @.omp.taskgraph.relocate.(ptr noundef %0, ptr noundef %1)
+// CHECK: [[REC:%.+]] = load ptr, ptr %.addr1,
+// CHECK-NEXT: [[TASK:%.+]] = load ptr, ptr %.addr,
+// CHECK-NEXT: [[SHAREDS:%.+]] = load ptr, ptr [[TASK]],
+// CHECK-NEXT: [[THISF:%.+]] = getelementptr inbounds nuw %struct.anon, ptr [[REC]], i32 0, i32 0
+// CHECK-NEXT: [[THIS:%.+]] = load ptr, ptr [[THISF]],
+// CHECK-NEXT: [[SLOT:%.+]] = getelementptr ptr, ptr [[SHAREDS]], i64 0
+// CHECK-NEXT: store ptr [[THIS]], ptr [[SLOT]],
+// CHECK: [[PRIV:%.+]] = getelementptr inbounds nuw %struct.kmp_task_t_with_privates, ptr %{{.+}}, i32 0, i32 1
+// CHECK-NEXT: [[THISF2:%.+]] = getelementptr inbounds nuw %struct.anon, ptr [[REC]], i32 0, i32 0
+// CHECK-NEXT: [[THIS2:%.+]] = load ptr, ptr [[THISF2]],
+// CHECK-NEXT: [[MADDR:%.+]] = getelementptr inbounds nuw %struct.S, ptr [[THIS2]], i32 0, i32 0
+// CHECK-NEXT: [[MDST:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[PRIV]], i32 0, i32 0
+// CHECK-NEXT: [[MVAL:%.+]] = load i32, ptr [[MADDR]],
+// CHECK-NEXT: store i32 [[MVAL]], ptr [[MDST]],
+// CHECK-NEXT: ret void
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_task_replay_copies_codegen.cpp b/clang/test/OpenMP/taskgraph_task_replay_copies_codegen.cpp
new file mode 100644
index 00000000000000..3081f868722d1c
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_task_replay_copies_codegen.cpp
@@ -0,0 +1,114 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// A replay generates its tasks afresh (OpenMP 6.0 [14.3]), each with private
+// copies of its own, constructed as the clauses say and destroyed when the
+// task completes. The body of a task that may be recorded constructs its
+// copies itself, from the initial values of its firstprivates that the task
+// descriptor holds, and libomp reuses the recorded descriptor, so:
+//
+// - the initial value of a firstprivate of a variable in the taskgraph's
+// enclosing data environment is copy-constructed from that variable, read
+// through the taskgraph's record, by the task's relocation helper when a
+// replay starts, and destroyed by the task's destructors thunk when it
+// ends;
+// - that of a `saved` firstprivate is the saved data environment: the copy
+// the task was recorded with, which the thunk destroys when the record is
+// freed;
+// - a private copy needs nothing: the body default-constructs it.
+//
+// So the thunk takes which initial values to destroy, and the task's flags
+// say so (0x200, here with tiedness and the thunk's own 0x8: 521).
+//
+// libomp keeps a copy of a recorded task, which a clone function completes by
+// copy-constructing the saved initial values; the relocation helper constructs
+// the others when a replay starts, so the clone leaves them alone. The copy
+// of the taskloop's tasks is the one task_dup makes.
+
+struct NonTrivial {
+ int v;
+ int *self;
+ NonTrivial();
+ NonTrivial(const NonTrivial &other);
+ ~NonTrivial();
+};
+
+void run() {
+ NonTrivial a, s, p;
+#pragma omp taskgraph
+ {
+#pragma omp task firstprivate(a) firstprivate(saved : s) private(p)
+ {
+ (void)(a.v + s.v + p.v);
+ }
+#pragma omp taskloop firstprivate(a) firstprivate(saved : s) num_tasks(2)
+ for (int i = 0; i < 2; ++i)
+ (void)(a.v + s.v);
+ }
+}
+
+// CHECK: call i32 @__kmpc_taskgraph_task({{.*}}, i32 521, {{.*}}, ptr @.omp.taskgraph.relocate., ptr @.omp_taskgraph_clone.)
+// CHECK: call i32 @__kmpc_taskgraph_taskloop({{.*}}, i32 521, {{.*}}, ptr @.omp_task_dup., ptr @.omp.taskgraph.relocate.[[LOOP:[.0-9]+]])
+
+// The task's privates are p, the initial values of a and s, and the body's
+// copies of a and s, in that order.
+
+// The clone copy-constructs the initial value of s, the saved one, and not
+// that of a.
+// CHECK: define internal void @.omp_taskgraph_clone.(ptr noundef %0, ptr noundef %1, i32 noundef %2)
+// CHECK: [[DPRIV:%.+]] = getelementptr inbounds nuw %struct.kmp_task_t_with_privates, ptr %{{.+}}, i32 0, i32 1
+// CHECK: [[SPRIV:%.+]] = getelementptr inbounds nuw %struct.kmp_task_t_with_privates, ptr %{{.+}}, i32 0, i32 1
+// CHECK-NEXT: [[SSRC:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[SPRIV]], i32 0, i32 2
+// CHECK-NEXT: [[SDST:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[DPRIV]], i32 0, i32 2
+// CHECK-NEXT: call void @_ZN10NonTrivialC1ERKS_(ptr {{.*}} [[SDST]], ptr {{.*}} [[SSRC]])
+// CHECK-NEXT: ret void
+
+// The thunk destroys both initial values for a task that runs outside a
+// record (0), a when a replay ends (1), and s when the record is freed (2).
+// CHECK: define internal noundef i32 @.omp_task_destructor.(i32 noundef %0, ptr noalias noundef %1, i32 noundef %2)
+// CHECK: [[PRIV:%.+]] = getelementptr inbounds nuw %struct.kmp_task_t_with_privates, ptr %{{.+}}, i32 0, i32 1
+// CHECK: switch i32 %{{.+}}, label %omp.destroy.done [
+// CHECK-NEXT: i32 0, label %omp.destroy.all
+// CHECK-NEXT: i32 1, label %omp.destroy.replay
+// CHECK-NEXT: i32 2, label %omp.destroy.record
+// CHECK-NEXT: ]
+// CHECK: omp.destroy.all:
+// CHECK-NEXT: [[A0:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[PRIV]], i32 0, i32 1
+// CHECK-NEXT: call void @_ZN10NonTrivialD1Ev(ptr {{.*}} [[A0]])
+// CHECK-NEXT: [[S0:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[PRIV]], i32 0, i32 2
+// CHECK-NEXT: call void @_ZN10NonTrivialD1Ev(ptr {{.*}} [[S0]])
+// CHECK-NEXT: br label %omp.destroy.done
+// CHECK: omp.destroy.replay:
+// CHECK-NEXT: [[A1:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[PRIV]], i32 0, i32 1
+// CHECK-NEXT: call void @_ZN10NonTrivialD1Ev(ptr {{.*}} [[A1]])
+// CHECK-NEXT: br label %omp.destroy.done
+// CHECK: omp.destroy.record:
+// CHECK-NEXT: [[S2:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[PRIV]], i32 0, i32 2
+// CHECK-NEXT: call void @_ZN10NonTrivialD1Ev(ptr {{.*}} [[S2]])
+// CHECK-NEXT: br label %omp.destroy.done
+
+// The relocation helper reads a through the record when a replay starts.
+// CHECK: define internal void @.omp.taskgraph.relocate.(ptr noundef %0, ptr noundef %1)
+// CHECK: [[REC:%.+]] = load ptr, ptr %.addr1,
+// CHECK: [[RPRIV:%.+]] = getelementptr inbounds nuw %struct.kmp_task_t_with_privates, ptr %{{.+}}, i32 0, i32 1
+// CHECK-NEXT: [[AF:%.+]] = getelementptr inbounds nuw %struct.anon, ptr [[REC]], i32 0, i32 0
+// CHECK-NEXT: [[AADDR:%.+]] = load ptr, ptr [[AF]],
+// CHECK-NEXT: [[ADST2:%.+]] = getelementptr inbounds nuw %struct..kmp_privates.t, ptr [[RPRIV]], i32 0, i32 1
+// CHECK-NEXT: call void @_ZN10NonTrivialC1ERKS_(ptr {{.*}} [[ADST2]], ptr {{.*}} [[AADDR]])
+// CHECK-NEXT: ret void
+
+// The taskloop's thunk and helper treat each of its tasks' initial values the
+// same way.
+// CHECK: define internal noundef i32 @.omp_task_destructor.{{[.0-9]+}}(i32 noundef %0, ptr noalias noundef %1, i32 noundef %2)
+// CHECK: omp.destroy.replay:
+// CHECK: call void @_ZN10NonTrivialD1Ev(
+// CHECK: omp.destroy.record:
+// CHECK: call void @_ZN10NonTrivialD1Ev(
+// CHECK: define internal void @.omp.taskgraph.relocate.[[LOOP]](ptr noundef %0, ptr noundef %1)
+// CHECK: call void @_ZN10NonTrivialC1ERKS_(
+// CHECK: ret void
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_template_handle_codegen.cpp b/clang/test/OpenMP/taskgraph_template_handle_codegen.cpp
new file mode 100644
index 00000000000000..0874fe58edb033
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_template_handle_codegen.cpp
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+template <typename T>
+void templ_handle(T &x) {
+#pragma omp taskgraph
+ {
+#pragma omp task shared(x)
+ {
+ x += 1;
+ }
+ }
+}
+
+int main() {
+ int i = 0;
+ long l = 0;
+ templ_handle(i);
+ templ_handle(l);
+ return 0;
+}
+
+// CHECK-DAG: @[[H1:.omp.taskgraph.handle[^ ]*]] = internal global ptr null
+// CHECK-DAG: @[[H2:.omp.taskgraph.handle[^ ]*]] = internal global ptr null
+
+// CHECK-LABEL: define linkonce_odr {{.*}} @_Z12templ_handleIiEvRT_(
+// CHECK: call void @__kmpc_taskgraph(ptr {{[^,]+}}, i32 {{[^,]+}}, ptr @[[H1]], i64 0, i32 0, i32 0, ptr {{[^,]+}}, ptr {{[^)]+}})
+
+// CHECK-LABEL: define linkonce_odr {{.*}} @_Z12templ_handleIlEvRT_(
+// CHECK-NOT: ptr @[[H1]]
+// CHECK: call void @__kmpc_taskgraph(ptr {{[^,]+}}, i32 {{[^,]+}}, ptr @[[H2]], i64 0, i32 0, i32 0, ptr {{[^,]+}}, ptr {{[^)]+}})
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_templates_codegen.cpp b/clang/test/OpenMP/taskgraph_templates_codegen.cpp
new file mode 100644
index 00000000000000..dc094c873ae805
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_templates_codegen.cpp
@@ -0,0 +1,77 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// Exercises taskgraph codegen with orthogonal language/runtime features:
+// - C++ templates + a non-trivial firstprivate copy on replay.
+// - taskloop reduction inside taskgraph (taskgraph reduction-init path).
+// - taskwait depend(...) inside taskgraph (task-generating path).
+
+template <typename T>
+struct Box {
+ T v;
+ Box();
+ Box(const Box &);
+ ~Box();
+};
+
+template <typename T>
+void templated_clone(T seed) {
+ Box<T> B;
+ B.v = seed;
+#pragma omp taskgraph
+ {
+#pragma omp task firstprivate(B)
+ {
+ (void)B.v;
+ }
+
+#pragma omp task replayable(false) firstprivate(B)
+ {
+ (void)B.v;
+ }
+ }
+}
+
+template <typename T>
+T templated_task_reduction(T seed) {
+ T Acc = seed;
+#pragma omp taskgraph
+ {
+ // A taskgroup is not task-generating, so it cannot be written in a
+ // taskgraph region; the reduction clause on taskloop reaches the same
+ // taskgraph reduction init through the taskgroup that codegen emits for it.
+#pragma omp taskloop reduction(+: Acc)
+ for (int i = 0; i < 4; ++i)
+ Acc += seed;
+#pragma omp taskwait depend(in: Acc)
+ }
+ return Acc;
+}
+
+int main() {
+ templated_clone<int>(7);
+ return templated_task_reduction<int>(1);
+}
+
+// The taskgraph task entry uses the taskgraph runtime path. Each replay
+// copy-constructs the non-trivial firstprivate's initial value afresh, in the
+// task's relocation helper. It is not a saved one, so the record's copy of the
+// task needs no clone function to construct it.
+// CHECK: call i32 @__kmpc_taskgraph_task(
+// CHECK-SAME: ptr @.omp.taskgraph.relocate., ptr null)
+// CHECK-NOT: define internal void @.omp_taskgraph_clone.(
+// CHECK: define internal void @.omp.taskgraph.relocate.(
+// CHECK: call void @_ZN3BoxIiEC1ERKS0_(
+
+// task_reduction in a taskgraph uses the dedicated taskgraph reduction init.
+// CHECK: call ptr @__kmpc_taskgraph_taskred_init(
+
+// taskwait depend(...) inside taskgraph uses the dedicated taskgraph taskwait
+// entry point instead of the generic taskwait runtime path.
+// CHECK: call void @__kmpc_taskgraph_taskwait(
+
+#endif
diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index d6d1b8420119c4..22701e817cf3f0 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -493,6 +493,7 @@ SetNumTeamsInParOrTeamsRegion "%1$s() called inside a parallel or teams region;
SetTeamsThreadLimitInParOrTeamsRegion "%1$s() called inside a parallel or teams region; call ignored."
MalformedDeviceEnvVar "%1$s: malformed device-scope environment variable name; expected `_ALL`, `_DEV`, or `_DEV_<n>` with a non-negative integer device id; ignored."
DeviceEnvVarOnGlobalScope "%1$s: device-scope environment variable suffix (`_ALL`/`_DEV`/`_DEV_<n>`) is not allowed on this base name (global-scope ICV); ignored."
+OmpTaskgraphBadCapture "Cannot locate captured shared variable reference for taskgraph replay"
OmpTaskgraphConcurrentRecord "%1$s: multiple threads attempting to re-record taskgraph concurrently for graph_id %2$d"
OmpTaskgraphDetachableTask "%1$s: a detachable task cannot be recorded for taskgraph replay"
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index b4697fa67f2db6..a82925fc7a13e1 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2476,6 +2476,18 @@ typedef kmp_int32 (*kmp_routine_entry_t)(kmp_int32, void *);
// are in the taskgraph's enclosing data environment. The task's body
// constructs its private copies from its initial values.
typedef void (*kmp_task_relocate_t)(struct kmp_task *, void *);
+// The destructors thunk of a task whose td_flags.taskgraph_destructors is set,
+// a task that may be recorded into a taskgraph. Its third argument says which
+// of the task's initial values to destroy: all of them, for a task that runs
+// outside a record; those a replay reads afresh, when a replay ends or a
+// taskloop's task is recorded; or the saved data environment, when the record
+// is freed.
+typedef kmp_int32 (*kmp_taskgraph_destructors_t)(kmp_int32, void *, kmp_int32);
+enum {
+ KMP_TASKGRAPH_DESTROY_ALL = 0,
+ KMP_TASKGRAPH_DESTROY_REPLAY = 1,
+ KMP_TASKGRAPH_DESTROY_RECORD = 2
+};
typedef union kmp_cmplrdata {
kmp_int32 priority; /**< priority specified by user for the task */
@@ -2704,6 +2716,7 @@ typedef struct kmp_taskgraph_node {
// encountering task until they complete -- e.g. a taskwait without 'nowait'.
// Everything recorded after one is ordered after it.
bool undeferred;
+ kmp_task_relocate_t relocate;
kmp_taskgraph_reduce_input_data_t *reduce_input;
kmp_depend_info_t *dep_list;
kmp_int32 ndeps;
@@ -2791,6 +2804,7 @@ typedef struct kmp_taskgraph_record {
kmp_int32 num_mutexes = 0;
struct kmp_taskgraph_exec_descr *exec_descrs = nullptr;
kmp_size_t num_exec_descrs = 0;
+ void *taskgraph_args = nullptr;
// We need a taskgroup structure to keep track of recorded tasks. This is
// set to TRUE if the user requested "nogroup" on the taskgraph directive
// (then we can avoid blocking at the end of the taskgraph region on replay,
@@ -2869,7 +2883,8 @@ typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
unsigned tasking_ser : 1;
unsigned task_serial : 1;
unsigned tasktype : 1;
- unsigned reserved : 7;
+ unsigned reserved : 6;
+ unsigned taskgraph_destructors : 1;
unsigned transparent : 1;
unsigned free_agent_eligible : 1;
unsigned detachable : 1;
@@ -2895,7 +2910,11 @@ typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
unsigned free_agent_eligible : 1; /* set if task can be executed by a
free-agent thread */
unsigned transparent : 1; /* transparent task support (compiler uses this) */
- unsigned reserved : 7; /* reserved for compiler use */
+ unsigned taskgraph_destructors : 1; /* the destructors thunk takes a third
+ argument, which of the task's initial
+ values to destroy (a task that may be
+ recorded into a taskgraph) */
+ unsigned reserved : 6; /* reserved for compiler use */
/* Library flags */ /* Total library flags must be 16 bits */
unsigned tasktype : 1; /* task is either explicit(1) or implicit (0) */
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 0c398e0de66688..59786c4ff82a8c 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -783,6 +783,21 @@ __kmp_taskgraph_exec_descr_finish(kmp_int32 gtid, kmp_info_t *thread,
// template<ompt>: effectively ompt_enabled.enabled!=0
// the version with ompt=false is inlined, allowing to optimize away all ompt
// code in this case
+#if OMP_TASKGRAPH_EXPERIMENTAL
+// Run TASK's destructors thunk. A task that may be recorded into a taskgraph
+// has a thunk that takes which of its initial values to destroy, WHICH (see
+// kmp_taskgraph_destructors_t).
+static void __kmp_task_run_destructors(kmp_int32 gtid, kmp_task_t *task,
+ kmp_int32 which) {
+ kmp_routine_entry_t destr_thunk = task->data1.destructors;
+ KMP_ASSERT(destr_thunk);
+ if (KMP_TASK_TO_TASKDATA(task)->td_flags.taskgraph_destructors)
+ ((kmp_taskgraph_destructors_t)destr_thunk)(gtid, task, which);
+ else
+ destr_thunk(gtid, task);
+}
+#endif
+
template <bool ompt>
static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
kmp_taskdata_t *resumed_task) {
@@ -851,12 +866,23 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
placed here, since at this point other tasks might have been released
hence overlapping the destructor invocations with some other work in the
released tasks. The OpenMP spec is not specific on when the destructors
- are invoked, so we should be free to choose. */
+ are invoked, so we should be free to choose.
+
+ A task of a taskgraph record is run once per replay from the same
+ descriptor, which holds the initial values of its firstprivates; its body
+ destroys the copies it made of them. The initial values are destroyed
+ when the replay ends or the record is freed, depending on where they come
+ from (see kmp_taskgraph_destructors_t). */
+#if OMP_TASKGRAPH_EXPERIMENTAL
+ if (UNLIKELY(taskdata->td_flags.destructors_thunk) && !is_taskgraph)
+ __kmp_task_run_destructors(gtid, task, KMP_TASKGRAPH_DESTROY_ALL);
+#else
if (UNLIKELY(taskdata->td_flags.destructors_thunk)) {
kmp_routine_entry_t destr_thunk = task->data1.destructors;
KMP_ASSERT(destr_thunk);
destr_thunk(gtid, task);
}
+#endif
KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1);
@@ -2619,10 +2645,11 @@ __kmp_build_exec_descrs(kmp_info_t *thread, kmp_taskgraph_region_t *region,
/// Reset, reparent and regroup the recorded task TASK and re-invoke it.
-static void __kmp_omp_tg_task(kmp_int32 gtid, kmp_task_t *task,
+static void __kmp_omp_tg_task(kmp_int32 gtid, kmp_taskgraph_node_t *node,
kmp_taskgroup_t *taskgroup,
kmp_taskdata_t *parent_taskdata,
bool serialize_immediate) {
+ kmp_task_t *task = node->task;
kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
taskdata->td_parent = parent_taskdata;
@@ -2656,6 +2683,8 @@ static void __kmp_omp_tg_task(kmp_int32 gtid, kmp_task_t *task,
if (parent_taskdata->td_flags.tasktype == TASK_EXPLICIT)
KMP_ATOMIC_INC(&parent_taskdata->td_allocated_child_tasks);
+ // The task's captures were relocated when the replay started
+ // (__kmp_taskgraph_relocate_nodes).
__kmp_omp_task(gtid, task, false);
}
@@ -2680,7 +2709,7 @@ static void __kmp_taskgraph_exec_descr_start(kmp_int32 gtid, kmp_info_t *thread,
kmp_taskdata_t *current_taskdata = thread->th.th_current_task;
if (descr->region->mutexset)
__kmp_taskgraph_acquire_locks(gtid, descr->region->owner, descr->region);
- __kmp_omp_tg_task(gtid, node->task, taskgroup, current_taskdata, false);
+ __kmp_omp_tg_task(gtid, node, taskgroup, current_taskdata, false);
break;
}
case TASKGRAPH_REGION_PARALLEL:
@@ -3428,6 +3457,60 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
}
#if OMP_TASKGRAPH_EXPERIMENTAL
+/// Relocate the recorded tasks of TASKGRAPH against this replay's
+/// taskgraph_args.
+///
+/// All of it happens here, before the replay releases any task, and none of
+/// it as a task is dispatched. A replay re-encounters every recorded
+/// construct up front (OpenMP 6.0 [14.3]), so a firstprivate takes the value
+/// its variable has when the replay starts, and a list item's base pointer is
+/// re-read at the same point. Doing it at dispatch would instead observe
+/// whatever the task's predecessors in the graph had written by then.
+///
+/// FIXME: That is too early for a construct after a 'taskwait' with a 'depend'
+/// clause, which a replay should only encounter once the tasks the 'taskwait'
+/// waits for have completed.
+static void __kmp_taskgraph_relocate_nodes(kmp_taskgraph_record_t *taskgraph) {
+ void *taskgraph_args = taskgraph->taskgraph_args;
+ for (kmp_int32 i = 0; i < taskgraph->num_tasks; i++) {
+ kmp_taskgraph_node_t *node = &taskgraph->record_map[i];
+ if (!node->task)
+ continue;
+ if (node->relocate) {
+ // Rewrites capture-by-reference variables to point at the replaying
+ // frame, which need not be the one the graph was recorded from, and
+ // reads the initial values of the task's firstprivates from it.
+ node->relocate(node->task, taskgraph_args);
+ } else if (node->task->shareds != NULL) {
+ // A missing relocation callback is only fatal when there is a non-empty
+ // shareds payload that may contain by-reference captures needing remap.
+ KMP_FATAL(OmpTaskgraphBadCapture);
+ }
+ }
+}
+
+/// Destroy WHICH of the initial values of the firstprivates of TASK, a task of
+/// a taskgraph record (see kmp_taskgraph_destructors_t), if any need it.
+static void __kmp_taskgraph_destroy_initial(kmp_int32 gtid, kmp_task_t *task,
+ kmp_int32 which) {
+ kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+ if (!taskdata->td_flags.destructors_thunk)
+ return;
+ // Only a task that may be recorded can be, and its thunk takes WHICH.
+ KMP_DEBUG_ASSERT(taskdata->td_flags.taskgraph_destructors);
+ __kmp_task_run_destructors(gtid, task, which);
+}
+
+/// End a replay of TASKGRAPH, once everything it started has completed. The
+/// initial values each recorded task's relocation helper read for the replay
+/// are destroyed, whether or not the task ran.
+static void __kmp_taskgraph_end_replay(kmp_int32 gtid,
+ kmp_taskgraph_record_t *taskgraph) {
+ for (kmp_int32 i = 0; i < taskgraph->num_tasks; i++)
+ if (kmp_task_t *task = taskgraph->record_map[i].task)
+ __kmp_taskgraph_destroy_initial(gtid, task, KMP_TASKGRAPH_DESTROY_REPLAY);
+}
+
static void __kmp_replay_taskgraph(kmp_int32 gtid,
kmp_taskdata_t *current_taskdata,
kmp_taskgraph_record_t *taskgraph,
@@ -5283,6 +5366,7 @@ __kmp_taskgraph_node_alloc(kmp_taskgraph_record_t *rec, kmp_task_t *task,
new_task->task = task;
new_task->taskloop_task = false;
new_task->undeferred = false;
+ new_task->relocate = nullptr;
new_task->reduce_input = nullptr;
new_task->ndeps = 0;
new_task->dep_list = nullptr;
@@ -5320,7 +5404,8 @@ __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task, kmp_uint64 *lb,
void *task_dup
#if OMP_TASKGRAPH_EXPERIMENTAL
,
- kmp_taskgraph_record_t *taskgraph_rec = nullptr
+ kmp_taskgraph_record_t *taskgraph_rec = nullptr,
+ kmp_task_relocate_t relocate = nullptr
#endif
) {
KMP_COUNT_BLOCK(OMP_TASKLOOP);
@@ -5416,7 +5501,13 @@ __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task, kmp_uint64 *lb,
taskgroup->taskgraph.reduce_input = nullptr;
}
node->taskloop_task = true;
+ node->relocate = relocate;
+ // The recording runs a taskloop of its own, so this task never runs
+ // until a replay. It keeps the initial values of its firstprivates that
+ // task_dup constructed, but those a replay reads afresh.
next_taskdata->owning_taskgraph = taskgraph_rec;
+ __kmp_taskgraph_destroy_initial(gtid, next_task,
+ KMP_TASKGRAPH_DESTROY_REPLAY);
// FIXME: These dependency fields might be back-filled by the as-yet
// unimplemented task_iteration subsidiary directive. We'll need a way
// to locate the correct task given the value of the iteration variable,
@@ -5705,7 +5796,8 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
int modifier, void *task_dup
#if OMP_TASKGRAPH_EXPERIMENTAL
,
- kmp_taskgraph_record_t *taskgraph_rec = nullptr
+ kmp_taskgraph_record_t *taskgraph_rec = nullptr,
+ kmp_task_relocate_t relocate = nullptr
#endif
) {
kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
@@ -5839,7 +5931,7 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
#if OMPT_SUPPORT
OMPT_GET_RETURN_ADDRESS(0),
#endif
- task_dup, taskgraph_rec);
+ task_dup, taskgraph_rec, relocate);
// check if clause value next
// Also require GOMP_taskloop to reduce to linear
// (taskdata->td_flags.native)
@@ -6022,6 +6114,7 @@ static void __kmp_taskgraph_reset(kmp_taskgraph_record_t *rec, kmp_int32 gtid,
rec->node_mutexes = nullptr;
rec->exec_descrs = nullptr;
rec->num_exec_descrs = 0;
+ rec->taskgraph_args = nullptr;
rec->next = nullptr;
}
@@ -6103,7 +6196,12 @@ static void __kmp_taskgraph_free(kmp_int32 gtid, kmp_taskgraph_record_t *rec,
// recorded by __kmpc_taskgraph_taskwait).
if (rec->record_map[task].task == nullptr)
continue;
- kmp_taskdata *taskdata = KMP_TASK_TO_TASKDATA(rec->record_map[task].task);
+ kmp_task_t *taskptr = rec->record_map[task].task;
+ kmp_taskdata *taskdata = KMP_TASK_TO_TASKDATA(taskptr);
+ // The task holds the saved data environment: the initial values of its
+ // firstprivates that every replay starts from.
+ __kmp_taskgraph_destroy_initial(gtid, taskptr,
+ KMP_TASKGRAPH_DESTROY_RECORD);
// Setting this here keeps an assertion in __kmp_free_task happy: the
// clone may never have been replayed, in which case 'complete' will be
// zero here, as initialized.
@@ -6150,19 +6248,19 @@ static kmp_taskgraph_header_t *__kmp_taskgraph_header_alloc(kmp_int32 gtid) {
}
// Clone a (new) task that has had its private variables and shared variables
-// initialised already.
-static kmp_task_t *__kmp_taskgraph_clone_task(kmp_info_t *thread,
- kmp_taskgraph_record_t *taskgraph,
- kmp_task_t *orig,
- size_t sizeof_kmp_task_t,
- size_t sizeof_shareds) {
- // FIXME: This should use a "taskdup" function like taskloops in cases where
- // private variables are not trivially copyable. For now, do it by plain
- // bitwise copy.
- // FIXME 2: It's intended that this copy be persistent, and can be
- // re-executed on taskgraph replay. Make sure that works (for shared
- // variables) if stack addresses change (i.e. a task-generating function is
- // called from different call stack depths).
+// initialised already, to be the task a replay runs.
+//
+// The clone holds the initial values of the task's firstprivates, which a
+// replay's body constructs its own copies from. Those of the saved data
+// environment live as long as the record: the memcpy below copies the ones
+// that are trivially copyable, and TASK_DUP, non-null when there are others,
+// copy-constructs those in place. The other initial values are left as the
+// memcpy's bytes, which hold no object: the task's relocation helper constructs
+// them when a replay starts, and they are destroyed when it ends. The
+// descriptor's other private storage holds no objects between replays.
+static kmp_task_t *__kmp_taskgraph_clone_task(
+ kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph, kmp_task_t *orig,
+ size_t sizeof_kmp_task_t, size_t sizeof_shareds, void *task_dup) {
kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(orig);
size_t shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(kmp_uint64));
@@ -6176,6 +6274,8 @@ static kmp_task_t *__kmp_taskgraph_clone_task(kmp_info_t *thread,
// New task's shared data has now moved. Update the pointer.
copy_task->shareds = (void *)((char *)copy_td + shareds_offset);
}
+ if (task_dup)
+ ((p_task_dup_t)task_dup)(copy_task, orig, /*lastpriv=*/0);
KMP_ATOMIC_ST_RLX(©_td->td_incomplete_child_tasks, 0);
return KMP_TASKDATA_TO_TASK(copy_td);
}
@@ -6330,8 +6430,15 @@ void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
fprintf(stderr, "Replay taskgraph %p from task %p\n", record,
KMP_TASKDATA_TO_TASK(current_taskdata));
__kmp_acquire_lock(&record->map_lock, gtid);
+ // Args must be captured under the above lock, else we can hit a race
+ // condition here. This is used to relocate host addresses in the
+ // taskgraph's initiating context, i.e. thread/stack frame., which may be
+ // different from the addresses seen at record time.
+ record->taskgraph_args = args;
+ __kmp_taskgraph_relocate_nodes(record);
__kmp_replay_taskgraph(gtid, current_taskdata, record, graph_id, taskgroup);
__kmpc_end_taskgroup(loc_ref, gtid);
+ __kmp_taskgraph_end_replay(gtid, record);
__kmp_release_lock(&record->map_lock, gtid);
// Decrement usage count and expire our record if it was 'graph_reset' by
// some thread during this replay.
@@ -6381,8 +6488,10 @@ kmp_uint32 __kmpc_taskgraph_task(ident_t *loc_ref, kmp_int32 gtid,
KMP_FATAL(OmpTaskgraphDetachableTask, src_loc);
__kmp_str_free(&src_loc);
}
+ // The record keeps the saved data environment; a replay reads the rest
+ // of the initial values afresh, so the clone does not construct them.
kmp_task_t *cloned_task = __kmp_taskgraph_clone_task(
- thread, rec, new_task, sizeof_kmp_task_t, sizeof_shareds);
+ thread, rec, new_task, sizeof_kmp_task_t, sizeof_shareds, task_dup);
kmp_taskgraph_node_t *node = __kmp_taskgraph_node_alloc(rec, cloned_task);
if (taskgroup->taskgraph.reduce_input) {
node->reduce_input = taskgroup->taskgraph.reduce_input;
@@ -6408,6 +6517,7 @@ kmp_uint32 __kmpc_taskgraph_task(ident_t *loc_ref, kmp_int32 gtid,
node->dep_list = (kmp_depend_info_t *)__kmp_thread_malloc(
thread, ndeps * sizeof(kmp_depend_info_t));
KMP_MEMCPY(node->dep_list, dep_list, ndeps * sizeof(kmp_depend_info_t));
+ node->relocate = relocate;
} else if (status == KMP_TDG_READY) {
#ifdef DEBUG_TASKGRAPH
fprintf(stderr,
@@ -6481,7 +6591,7 @@ kmp_uint32 __kmpc_taskgraph_taskloop(
kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&rec->status);
if (status == KMP_TDG_RECORDING)
__kmp_taskloop(loc_ref, gtid, new_task, if_val, lb, ub, st, nogroup,
- sched, grainsize, modifier, task_dup, rec);
+ sched, grainsize, modifier, task_dup, rec, relocate);
else if (status == KMP_TDG_READY) {
#ifdef DEBUG_TASKGRAPH
fprintf(stderr, "non-taskgraph taskloop entry point for taskloop in "
diff --git a/openmp/runtime/test/taskgraph/taskgraph_concurrent_replay_relocation.cpp b/openmp/runtime/test/taskgraph/taskgraph_concurrent_replay_relocation.cpp
new file mode 100644
index 00000000000000..8d5dc9090a446c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_concurrent_replay_relocation.cpp
@@ -0,0 +1,79 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A taskgraph inside a parallel region with no enclosing single -- an ordinary,
+// spec-legal program -- has every thread encounter the directive and replay the
+// same shared record. Each encounter passes its own outlined-entry args, which
+// replay uses to relocate the recorded tasks' by-reference captures onto the
+// encountering thread's frame.
+//
+// map_lock serialises the replay itself: the encountering thread holds it until
+// the implicit taskgroup has waited the whole graph out. The args, though,
+// were published before that lock was taken, so a thread queued up behind an
+// in-flight replay would retarget relocation at its own frame mid-replay, and
+// every task still to be issued would write through to another thread's locals.
+// Silent wrong data, no crash.
+//
+// The chain of dependent tasks matters: a graph whose tasks are all immediately
+// runnable is relocated entirely by the encountering thread, whereas a
+// successor is relocated from the task-completion hook, on whichever worker
+// finished its predecessor. That is also why the args cannot simply be passed
+// down the replay call chain -- relocation does not run on the thread that has
+// them.
+//
+// Racy by nature, so a pass is not a proof; the point is that it reports
+// hundreds of wrong values against the unfixed runtime.
+
+#include <cstdio>
+#include <omp.h>
+
+static constexpr int Threads = 32;
+static constexpr int Reps = 300;
+static constexpr int Chain = 6;
+
+int main() {
+ int wrong = 0, total = 0;
+
+#pragma omp parallel num_threads(Threads) reduction(+ : wrong, total)
+ {
+ const int me = omp_get_thread_num();
+ // Captured by reference by the recorded tasks, so reaching it on replay
+ // depends on relocation having this encounter's args.
+ const int local = me * 1000;
+
+ for (int rep = 0; rep < Reps; ++rep) {
+ int got[Chain];
+ for (int i = 0; i < Chain; ++i)
+ got[i] = -1;
+ // Serialises the tasks against each other, so that all but the first are
+ // released from the completion hook rather than at replay start.
+ int chain = 0;
+
+#pragma omp taskgraph
+ {
+ for (int i = 0; i < Chain; ++i) {
+#pragma omp task shared(local, got, chain) depend(inout : chain)
+ got[i] = local + i;
+ }
+ }
+
+ for (int i = 0; i < Chain; ++i) {
+ ++total;
+ if (got[i] != me * 1000 + i)
+ ++wrong;
+ }
+ }
+ }
+
+ if (wrong) {
+ std::printf("FAIL %d of %d replayed tasks relocated onto the wrong frame\n",
+ wrong, total);
+ return 1;
+ }
+ std::printf("PASS %d replays\n", total);
+ return 0;
+}
+
+// CHECK: PASS
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp
index fd2323c21fe5f4..b5008095e6f2f9 100644
--- a/openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp
@@ -36,7 +36,9 @@ int main()
res += arr[j];
}
}
- #pragma omp taskloop reduction(+: res2) num_tasks(10)
+ // A replay takes firstprivate values when it starts, before the
+ // first taskloop has combined 'res', so the result is read shared.
+ #pragma omp taskloop reduction(+: res2) num_tasks(10) shared(res)
{
for (int j = 0; j < 10; j++) {
res2 += res * arr2[j];
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_later_writer.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_later_writer.cpp
new file mode 100644
index 00000000000000..2e26aea848790a
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_later_writer.cpp
@@ -0,0 +1,66 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A firstprivate of a non-trivially-copyable type, whose variable a task
+// generated after it writes before the first task runs. The task's copy is
+// initialized when the task is generated [7.5.4], and a replay generates the
+// recorded tasks in their recorded order (OpenMP 6.0 [14.3]), so on every
+// replay as on the recording the copy is the vector before the write -- even
+// though the write frees the vector's storage before the reading task's body
+// starts. The later task is ordered after the earlier one's generation, so
+// the program has no race.
+//
+// The gate task holds the reader back until the writer is done.
+
+#include <atomic>
+#include <cstdio>
+#include <vector>
+
+static std::atomic<int> Written;
+static size_t SeenSize[4];
+static int SeenFirst[4];
+static bool Opened[4];
+
+__attribute__((noinline)) static void run(int R) {
+ std::vector<int> V(R + 1, R);
+ int Gate = 0;
+ Written = 0;
+#pragma omp taskgraph graph_id(5)
+ {
+#pragma omp task depend(out : Gate) shared(Opened)
+ {
+ // Bounded, so that a schedule that runs the reader first cannot hang.
+ for (long I = 0; I < 2000000000L && !Written.load(); ++I)
+ ;
+ Opened[R] = Written.load();
+ }
+#pragma omp task depend(in : Gate) firstprivate(V) shared(SeenSize, SeenFirst)
+ {
+ SeenSize[R] = V.size();
+ SeenFirst[R] = V.empty() ? -1 : V[0];
+ }
+#pragma omp task shared(V)
+ {
+ V.assign(100, 7);
+ Written = 1;
+ }
+ }
+}
+
+int main() {
+#pragma omp parallel num_threads(4)
+#pragma omp single
+ for (int R = 0; R < 4; ++R)
+ run(R);
+ for (int R = 0; R < 4; ++R)
+ std::printf("run %d: size %zu, first %d, writer first %d\n", R, SeenSize[R],
+ SeenFirst[R], Opened[R]);
+ return 0;
+}
+
+// CHECK: run 0: size 1, first 0, writer first 1
+// CHECK-NEXT: run 1: size 2, first 1, writer first 1
+// CHECK-NEXT: run 2: size 3, first 2, writer first 1
+// CHECK-NEXT: run 3: size 4, first 3, writer first 1
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_nontrivial_replay.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_nontrivial_replay.cpp
new file mode 100644
index 00000000000000..076a9dfd00d323
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_nontrivial_replay.cpp
@@ -0,0 +1,77 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A firstprivate of a non-trivially-copyable type, on a replayed task. Each
+// replay generates the task again (OpenMP 6.0 [14.3]), so the task's copy is
+// copy-constructed [7.5.4] from the variable as it is when the replay starts
+// -- the replay's enclosing data environment being the taskgraph construct's
+// -- and destroyed when the task completes [7.5.3], like any task's. So:
+//
+// - the copy reflects the variable at each replay, not at the recording;
+// - the body's changes to its copy are gone by the next replay;
+// - the copy is destroyed before the taskgraph construct completes, while
+// the frame it was copied from is still live. Here its destructor reports
+// to a local of that frame;
+// - every copy made is destroyed.
+//
+// libomp makes more than one copy for an encounter: the task holds the
+// variable's value, which its body copies. So what is checked is that every
+// copy made from the frame is destroyed before the construct completes, not
+// how many there are.
+
+#include <cstdio>
+#include <vector>
+
+struct Counted {
+ static int Live;
+ int *Copies, *Destroyed;
+ std::vector<int> V;
+ Counted(int *C, int *D, int N) : Copies(C), Destroyed(D), V(N, 0) { ++Live; }
+ Counted(const Counted &O) : Copies(O.Copies), Destroyed(O.Destroyed), V(O.V) {
+ ++Live;
+ ++*Copies;
+ }
+ ~Counted() {
+ --Live;
+ ++*Destroyed;
+ }
+};
+int Counted::Live = 0;
+
+static size_t Seen[4];
+
+__attribute__((noinline)) static bool run(int R) {
+ int Copies = 0, Destroyed = 0;
+ Counted C(&Copies, &Destroyed, R + 1);
+#pragma omp taskgraph graph_id(2)
+ {
+#pragma omp task firstprivate(C) shared(Seen)
+ {
+ Seen[R] = C.V.size();
+ C.V.push_back(R);
+ }
+ }
+ // C itself is destroyed after this is read.
+ return Copies > 0 && Destroyed == Copies;
+}
+
+int main() {
+ bool AllDestroyed[4];
+#pragma omp parallel
+#pragma omp single
+ for (int R = 0; R < 4; ++R)
+ AllDestroyed[R] = run(R);
+ for (int R = 0; R < 4; ++R)
+ std::printf("run %d: size %zu, copies destroyed %d\n", R, Seen[R],
+ AllDestroyed[R]);
+ std::printf("live %d\n", Counted::Live);
+ return 0;
+}
+
+// CHECK: run 0: size 1, copies destroyed 1
+// CHECK-NEXT: run 1: size 2, copies destroyed 1
+// CHECK-NEXT: run 2: size 3, copies destroyed 1
+// CHECK-NEXT: run 3: size 4, copies destroyed 1
+// CHECK-NEXT: live 0
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_refresh.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_refresh.cpp
new file mode 100644
index 00000000000000..71fcc6df7ea8c8
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_refresh.cpp
@@ -0,0 +1,86 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A replay starts from the taskgraph construct's enclosing data environment
+// (OpenMP 6.0 [14.3]), so a firstprivate of a variable that environment has
+// takes the value the variable has at replay -- whether it is a scalar, a
+// struct or an array, an automatic variable of the enclosing function or one
+// with static storage duration, on a task or on each task of a taskloop. The
+// saved data environment supplies the rest: a list item with the `saved`
+// modifier, and a variable declared inside the region, keep the value they had
+// when the graph was recorded.
+
+#include <cstdio>
+
+struct Pt {
+ int X, Y;
+};
+
+static int G;
+static int Out[6];
+static int Loop[4];
+
+__attribute__((noinline)) static void run(int N, Pt P, int S) {
+ int A[3] = {N, N + 1, N + 2};
+#pragma omp taskgraph graph_id(7)
+ {
+ int In = N;
+#pragma omp task firstprivate(N, P, A) shared(Out)
+ {
+ Out[0] = N;
+ Out[1] = P.Y;
+ Out[2] = A[2];
+ }
+#pragma omp task firstprivate(G) shared(Out)
+ Out[3] = G;
+#pragma omp task firstprivate(saved : S) shared(Out)
+ Out[4] = S;
+#pragma omp task firstprivate(In) shared(Out)
+ Out[5] = In;
+#pragma omp taskloop firstprivate(N) num_tasks(4) shared(Loop)
+ for (int I = 0; I < 4; ++I)
+ Loop[I] = N + I;
+ }
+}
+
+// Replays from Depth frames further down, so the enclosing function's
+// variables are not where they were when the graph was recorded.
+__attribute__((noinline)) static void deeper(int Depth, int N, Pt P, int S) {
+ volatile int Padding[64];
+ for (int I = 0; I < 64; ++I)
+ Padding[I] = N + I;
+ if (Depth)
+ deeper(Depth - 1, N, P, S);
+ else
+ run(N, P, S);
+}
+
+int main() {
+ int Errors = 0;
+#pragma omp parallel num_threads(4)
+#pragma omp single
+ for (int It = 1; It <= 4; ++It) {
+ G = 10 * It;
+ deeper(It, It, Pt{0, 20 * It}, 30 * It);
+ const int Want[6] = {It, 20 * It, It + 2, 10 * It, 30, 1};
+ for (int K = 0; K < 6; ++K)
+ if (Out[K] != Want[K]) {
+ std::printf("FAIL replay %d Out[%d] %d want %d\n", It, K, Out[K],
+ Want[K]);
+ ++Errors;
+ }
+ for (int K = 0; K < 4; ++K)
+ if (Loop[K] != It + K) {
+ std::printf("FAIL replay %d Loop[%d] %d want %d\n", It, K, Loop[K],
+ It + K);
+ ++Errors;
+ }
+ }
+ std::printf("taskgraph firstprivate refresh errors: %d\n", Errors);
+ return Errors != 0;
+}
+
+// CHECK-NOT: FAIL
+// CHECK: taskgraph firstprivate refresh errors: 0
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_replay_start.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_replay_start.cpp
new file mode 100644
index 00000000000000..97309ad22315ed
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_replay_start.cpp
@@ -0,0 +1,58 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A replay re-encounters every recorded construct before any of its tasks
+// runs (OpenMP 6.0 [14.3]), so a firstprivate takes the value its variable has
+// when the replay starts. A write to the variable by another task of the same
+// graph is not seen, even by a task that depends on the writer and so only
+// starts once the write has happened.
+//
+// Only the replays are checked: when the graph is recorded, the writer may
+// run before the reader is encountered, and then the reader sees the write.
+
+#include <cstdio>
+
+static int G;
+static int Out[2];
+
+__attribute__((noinline)) static void run(int N) {
+#pragma omp taskgraph graph_id(11)
+ {
+#pragma omp task shared(N) depend(out : N)
+ {
+ N = -1;
+ G = -1;
+ }
+#pragma omp task firstprivate(N, G) shared(Out) depend(in : N)
+ {
+ Out[0] = N;
+ Out[1] = G;
+ }
+ }
+}
+
+int main() {
+ int Errors = 0;
+#pragma omp parallel num_threads(4)
+#pragma omp single
+ for (int It = 1; It <= 4; ++It) {
+ G = 10 * It;
+ run(It);
+ if (It == 1)
+ continue;
+ const int Want[2] = {It, 10 * It};
+ for (int K = 0; K < 2; ++K)
+ if (Out[K] != Want[K]) {
+ std::printf("FAIL replay %d Out[%d] %d want %d\n", It, K, Out[K],
+ Want[K]);
+ ++Errors;
+ }
+ }
+ std::printf("taskgraph firstprivate replay start errors: %d\n", Errors);
+ return Errors != 0;
+}
+
+// CHECK-NOT: FAIL
+// CHECK: taskgraph firstprivate replay start errors: 0
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_replay_writes.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_replay_writes.cpp
new file mode 100644
index 00000000000000..bdfb2554fa39df
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_replay_writes.cpp
@@ -0,0 +1,62 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A replay generates its tasks afresh (OpenMP 6.0 [14.3]), each with private
+// copies of its own [7.5.3], so a task body that writes its copy of a
+// firstprivate affects that replay's task alone. That holds whatever the copy
+// is initialized from: a variable of the taskgraph's enclosing data
+// environment, or the saved data environment -- a `saved` list item, or a
+// variable declared inside the region -- on a task or on a task of a taskloop.
+//
+// The replayed task descriptor is the recorded one, so a write that reached
+// the descriptor rather than a copy of the replay's own would be seen by the
+// next replay, from the second replay on.
+
+#include <cstdio>
+
+static int Out[4][4];
+
+__attribute__((noinline)) static void run(int R, int X, int S) {
+#pragma omp taskgraph graph_id(1)
+ {
+ int L = 30;
+#pragma omp task firstprivate(X) shared(Out)
+ {
+ Out[R][0] = X;
+ X += 100;
+ }
+#pragma omp task firstprivate(saved : S) shared(Out)
+ {
+ Out[R][1] = S;
+ S += 100;
+ }
+#pragma omp task firstprivate(L) shared(Out)
+ {
+ Out[R][2] = L;
+ L += 100;
+ }
+#pragma omp taskloop firstprivate(saved : S) shared(Out) num_tasks(1)
+ for (int I = 0; I < 1; ++I) {
+ Out[R][3] = S;
+ S += 100;
+ }
+ }
+}
+
+int main() {
+#pragma omp parallel
+#pragma omp single
+ for (int R = 0; R < 4; ++R)
+ run(R, 10, 20 + R);
+ for (int R = 0; R < 4; ++R)
+ std::printf("run %d: %d %d %d %d\n", R, Out[R][0], Out[R][1], Out[R][2],
+ Out[R][3]);
+ return 0;
+}
+
+// CHECK: run 0: 10 20 30 20
+// CHECK-NEXT: run 1: 10 20 30 20
+// CHECK-NEXT: run 2: 10 20 30 20
+// CHECK-NEXT: run 3: 10 20 30 20
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial.cpp
new file mode 100644
index 00000000000000..3404f13873e6f3
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial.cpp
@@ -0,0 +1,140 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Verifies the constructor/destructor lifecycle of a non-trivially-copyable
+// 'firstprivate(saved: ...)' list item on a replayable task.
+//
+// The saved data environment keeps a copy of the list item, made when the
+// task is recorded, for the life of the taskgraph record (OpenMP 6.0 [14.3]).
+// Each replay generates the task again, with a new private copy [7.5.3]
+// initialized from that saved copy by the copy constructor [7.5.4] and
+// destroyed when the task's body ends. The destructor here writes a
+// recognisable sentinel value (-1), so a replay that saw a destroyed saved
+// copy would observe -1 rather than the saved value.
+//
+// Construction/destruction balance is verified at program exit via a
+// global guard.
+
+#include <cstdio>
+#include <cstdlib>
+
+struct Tracker {
+ static int Ctors;
+ static int CopyCtors;
+ static int Dtors;
+
+ int Value;
+
+ explicit Tracker(int V) : Value(V) { ++Ctors; }
+ Tracker(const Tracker &Other) : Value(Other.Value) { ++CopyCtors; }
+ // Use a sentinel value on destruction so that any read-after-destroy of
+ // the saved snapshot becomes observable.
+ ~Tracker() {
+ ++Dtors;
+ Value = -1;
+ }
+};
+
+int Tracker::Ctors = 0;
+int Tracker::CopyCtors = 0;
+int Tracker::Dtors = 0;
+
+// 'Failed' is shared across tasks and threads so we can record a fail
+// status from within the task body and across the program-exit guard.
+static int Failed = 0;
+
+__attribute__((noinline)) static void run_taskgraph_nontrivial(int seed) {
+ Tracker Local(seed);
+ int observed = 0;
+
+#pragma omp taskgraph graph_id(927)
+ {
+#pragma omp task firstprivate(saved : Local) shared(observed)
+ {
+ // Each replay must observe the value captured at recording time
+ // (which was 11), never the sentinel -1 written by
+ // Tracker::~Tracker().
+ observed = Local.Value;
+ }
+ }
+
+ if (observed != 11) {
+ std::fprintf(stderr, "FAIL replay observed=%d expected=11 seed=%d\n",
+ observed, seed);
+ Failed = 1;
+ }
+}
+
+// Final accounting fires at program exit. Copies are copy-constructed:
+// 1. For the task the recording execution runs, at task allocation, as the
+// initial value its body copies, and destroyed when that task finishes.
+// 2. For the record's copy of that task, from (1) when the task is
+// recorded: the saved data environment. It lives as long as the record,
+// which is freed by __kmp_taskgraph_free; that is not driven by program
+// exit in libomp today, hence the asymmetric expected dtor count below.
+// 3. For the recording execution's body, from (1), and destroyed when the
+// body ends.
+// 4. For each replay's body, from (2), and destroyed when the body ends.
+static const int Replays = 5;
+struct ExitGuard {
+ ~ExitGuard() {
+ bool ok = true;
+ if (Tracker::CopyCtors != 3 + Replays) {
+ std::fprintf(stderr,
+ "FAIL (exit) CopyCtors=%d expected=%d (two for the "
+ "recording, one for the saved data environment, one per "
+ "replay)\n",
+ Tracker::CopyCtors, 3 + Replays);
+ ok = false;
+ }
+ // Expected destruction count at program exit:
+ // - Local objects: one ctor + one dtor per call to
+ // run_taskgraph_nontrivial.
+ // - The recording execution's two copies and each replay's copy.
+ // - Not the saved copy (see above).
+ int expected_dtors = Tracker::Ctors + /* recording */ 2 + Replays;
+ if (Tracker::Dtors != expected_dtors) {
+ std::fprintf(stderr,
+ "FAIL (exit) ctor/dtor imbalance ctors=%d copyctors=%d "
+ "dtors=%d expected_dtors=%d\n",
+ Tracker::Ctors, Tracker::CopyCtors, Tracker::Dtors,
+ expected_dtors);
+ ok = false;
+ }
+
+ if (!ok || Failed) {
+ std::fprintf(stderr, "FAIL firstprivate(saved) non-trivial lifecycle\n");
+ std::_Exit(1);
+ }
+
+ std::fprintf(stderr,
+ "PASS firstprivate(saved) non-trivial lifecycle "
+ "ctors=%d copyctors=%d dtors=%d\n",
+ Tracker::Ctors, Tracker::CopyCtors, Tracker::Dtors);
+ }
+};
+
+// Global guard whose destructor fires at program exit, after all OpenMP
+// teardown has occurred and the taskgraph record has been freed.
+static ExitGuard Guard;
+
+int main() {
+ // Recording run. Inside the function we create Local(11), which the task's
+ // firstprivate copies, and the saved data environment copies that in turn.
+ run_taskgraph_nontrivial(11);
+
+ // Replay runs. Each replay's task copies the saved copy and destroys its
+ // own copy when its body ends. The body observes the saved value (11) on
+ // every replay -- the per-call seed argument is intentionally ignored by
+ // the task body to demonstrate that the saved data environment drives what
+ // the task sees, not the call-site argument.
+ for (int i = 0; i < Replays; ++i)
+ run_taskgraph_nontrivial(42 + i);
+
+ return 0;
+}
+
+// CHECK: PASS firstprivate(saved) non-trivial lifecycle
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial_selfref.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial_selfref.cpp
new file mode 100644
index 00000000000000..06bbf072907207
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial_selfref.cpp
@@ -0,0 +1,94 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Verifies that a non-trivially-copyable 'firstprivate(saved: ...)' list item
+// is copy-constructed, never bitwise-copied: into the saved data environment
+// when the task is recorded, and from there into each replay's task.
+//
+// 'SelfRef' carries an internal pointer that the constructor wires back to
+// its own member. A bitwise copy would leave the copy's internal pointer
+// aimed at the object it was copied from -- the recording's copy, which is
+// destroyed when the recording's task finishes, or the saved copy -- so a
+// replay would read a destroyed object or the wrong one.
+
+#include <cstdio>
+#include <cstdlib>
+
+struct SelfRef {
+ int Value;
+ // Self-pointer that the (copy) constructor steers at our own 'Value'
+ // member. Bitwise duplication leaves this dangling.
+ int *Inside;
+
+ explicit SelfRef(int V) : Value(V), Inside(&Value) {}
+ SelfRef(const SelfRef &Other) : Value(Other.Value), Inside(&Value) {}
+ ~SelfRef() {
+ // Poison the self pointer so any dangling read after destruction is
+ // observable as a (likely) crash or wrong value.
+ Inside = nullptr;
+ Value = -1;
+ }
+};
+
+static int Failed = 0;
+
+__attribute__((noinline)) static void run_taskgraph_selfref(int seed) {
+ SelfRef Local(seed);
+ int observed_via_self = 0;
+ int observed_value = 0;
+
+#pragma omp taskgraph graph_id(3142)
+ {
+#pragma omp task firstprivate(saved : Local) \
+ shared(observed_via_self, observed_value)
+ {
+ // Read through the self pointer. This is exactly the operation that
+ // a bitwise copy would break: 'Inside' would point at the object the
+ // copy was taken from, not at this copy's 'Value'.
+ observed_via_self = *Local.Inside;
+ observed_value = Local.Value;
+ }
+ }
+
+ if (observed_value != 7 || observed_via_self != 7) {
+ std::fprintf(stderr,
+ "FAIL seed=%d observed_value=%d observed_via_self=%d "
+ "(expected both = 7)\n",
+ seed, observed_value, observed_via_self);
+ Failed = 1;
+ }
+}
+
+struct ExitGuard {
+ ~ExitGuard() {
+ if (Failed) {
+ std::fprintf(stderr, "FAIL non-trivial taskgraph copy\n");
+ std::_Exit(1);
+ }
+ std::fprintf(stderr, "PASS non-trivial taskgraph copy\n");
+ }
+};
+
+static ExitGuard Guard;
+
+int main() {
+ // Recording copies Local(7) into the task's '.kmp_privates.t' slot, and
+ // from there into the saved data environment. After this returns 'Local'
+ // and the recording's copy are gone, so a saved 'Inside' pointer that still
+ // aimed at either would dangle.
+ run_taskgraph_selfref(7);
+
+ // Replays. Each one must observe the saved 7 both directly and through
+ // its own copy's self-referencing pointer. Different seed values are
+ // intentionally ignored by the task body: only the saved data environment
+ // drives what the task sees.
+ for (int i = 0; i < 5; ++i)
+ run_taskgraph_selfref(42 + i);
+
+ return 0;
+}
+
+// CHECK: PASS non-trivial taskgraph copy
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_static.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_static.cpp
new file mode 100644
index 00000000000000..ae613e2bebef24
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_static.cpp
@@ -0,0 +1,100 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// OpenMP 6.0 [14.3]: the 'saved' modifier on a 'firstprivate' clause of a
+// replayable construct extends the saved data environment to also include
+// copies of variables with static storage duration that appear in the
+// clause. Within the current Clang/libomp implementation those snapshots
+// live in the per-task '.kmp_privates.t' tail struct, which is allocated
+// once at recording time and reused for every replay of the recorded task.
+// This test exercises four flavours of static-storage list items and
+// verifies that the saved snapshot is what each replay observes, regardless
+// of any subsequent mutation of the underlying static variable between
+// taskgraph encounters.
+
+#include <cstdio>
+
+static int FileScopeStaticInt = 100;
+static const int FileScopeConstStaticInt = 200;
+
+struct WithStaticMember {
+ static int StaticMember;
+ static const int StaticConstMember = 400;
+};
+int WithStaticMember::StaticMember = 300;
+// Out-of-line definition is required (pre-C++17) because the saved
+// firstprivate slot captures the static by-value via odr-use.
+const int WithStaticMember::StaticConstMember;
+
+__attribute__((noinline)) static void
+run_taskgraph_saved_static(int *out_fs, int *out_fsc, int *out_local,
+ int *out_member, int *out_member_const) {
+ static int LocalStaticInt = 500;
+
+#pragma omp taskgraph graph_id(811)
+ {
+#pragma omp task firstprivate(saved : FileScopeStaticInt, \
+ FileScopeConstStaticInt, LocalStaticInt, \
+ WithStaticMember::StaticMember, \
+ WithStaticMember::StaticConstMember) \
+ shared(out_fs, out_fsc, out_local, out_member, out_member_const)
+ {
+ *out_fs = FileScopeStaticInt;
+ *out_fsc = FileScopeConstStaticInt;
+ *out_local = LocalStaticInt;
+ *out_member = WithStaticMember::StaticMember;
+ *out_member_const = WithStaticMember::StaticConstMember;
+ }
+ }
+}
+
+int main() {
+ bool failed = false;
+ int fs = -1, fsc = -1, local = -1, member = -1, member_const = -1;
+
+ // First call: recording. Each captured-by-saved static is snapshotted
+ // into the task's '.kmp_privates.t' slot at this point.
+ run_taskgraph_saved_static(&fs, &fsc, &local, &member, &member_const);
+ if (fs != 100 || fsc != 200 || local != 500 || member != 300 ||
+ member_const != 400) {
+ std::fprintf(stderr,
+ "FAIL initial record fs=%d fsc=%d local=%d member=%d "
+ "member_const=%d\n",
+ fs, fsc, local, member, member_const);
+ failed = true;
+ }
+
+ // Mutate the underlying non-const statics. Because the task's firstprivate
+ // slots were snapshotted with 'saved:' at recording, every subsequent
+ // replay must continue to observe the recorded values (100, 500, 300) for
+ // the non-const statics, not the mutated values.
+ FileScopeStaticInt = 11;
+ WithStaticMember::StaticMember = 13;
+ // LocalStaticInt is not visible here; we can rely on the fact that the
+ // function-local static is also snapshotted at recording.
+
+ for (int i = 0; i < 4; ++i) {
+ fs = fsc = local = member = member_const = -1;
+ run_taskgraph_saved_static(&fs, &fsc, &local, &member, &member_const);
+ if (fs != 100 || fsc != 200 || local != 500 || member != 300 ||
+ member_const != 400) {
+ std::fprintf(stderr,
+ "FAIL replay %d fs=%d fsc=%d local=%d member=%d "
+ "member_const=%d\n",
+ i, fs, fsc, local, member, member_const);
+ failed = true;
+ }
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr,
+ "PASS firstprivate(saved) statics persist across replays\n");
+ return 0;
+}
+
+// CHECK: PASS firstprivate(saved) statics persist across replays
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_stack_depth.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_stack_depth.cpp
new file mode 100644
index 00000000000000..ff706af8009177
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_stack_depth.cpp
@@ -0,0 +1,116 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+// Keep the observable result in stable storage so this test isolates
+// firstprivate replay across different stack depths. There is a separate bug
+// test for replayed tasks writing through a stack-local shared pointer cached
+// at record time.
+//
+// 'payload' and 'seed' are in the taskgraph's enclosing data environment, so
+// each replay takes their current values (OpenMP 6.0 [14.3]), read from the
+// current frame at whatever depth the replay is entered.
+static volatile int ReplayResult = -1;
+
+struct Payload {
+ int values[6];
+ int bias;
+};
+
+__attribute__((noinline)) static int evaluate_payload(const Payload &payload,
+ int seed) {
+ return seed * payload.values[0] - payload.values[1] +
+ payload.values[2] * payload.values[3] - payload.bias +
+ payload.values[4] - payload.values[5];
+}
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+ volatile int scratch[4096];
+
+ for (int i = 0; i < 4096; ++i)
+ scratch[i] = base + i;
+
+ StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+ Payload payload{
+ {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+ seed * 17 + 19};
+ ReplayResult = -1;
+
+#pragma omp taskgraph graph_id(91)
+ {
+#pragma omp task firstprivate(payload, seed) shared(ReplayResult)
+ {
+ ReplayResult = evaluate_payload(payload, seed);
+ }
+ }
+
+ return ReplayResult;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+ volatile int padding[128];
+
+ for (int i = 0; i < 128; ++i)
+ padding[i] = seed + depth + i;
+
+ StackSink += padding[(seed + depth) & 127];
+
+ if (depth == 0)
+ return run_taskgraph(seed);
+ return call_with_depth(seed, depth - 1);
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+ Payload payload{
+ {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+ seed * 17 + 19};
+ return evaluate_payload(payload, seed);
+}
+
+int main() {
+ constexpr int NumCalls = 4;
+ constexpr int Seeds[NumCalls] = {3, 17, 29, 41};
+ constexpr int Depths[NumCalls] = {0, 3, 1, 5};
+
+ int recorded = -1;
+ bool failed = false;
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ recorded = call_with_depth(Seeds[0], Depths[0]);
+ if (recorded != expected_result(Seeds[0])) {
+ std::fprintf(stderr, "FAIL initial record got=%d expected=%d\n",
+ recorded, expected_result(Seeds[0]));
+ failed = true;
+ }
+
+ for (int i = 1; i < NumCalls; ++i) {
+ clobber_stack(Seeds[i] * 1000);
+ const int replayed = call_with_depth(Seeds[i], Depths[i]);
+ if (replayed != expected_result(Seeds[i])) {
+ std::fprintf(
+ stderr, "FAIL replay %d depth=%d seed=%d got=%d expected=%d\n", i,
+ Depths[i], Seeds[i], replayed, expected_result(Seeds[i]));
+ failed = true;
+ }
+ }
+ }
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr, "PASS firstprivate stack result=%d\n", recorded);
+ return 0;
+}
+
+// CHECK: PASS firstprivate stack result=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_untied_replay.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_untied_replay.cpp
new file mode 100644
index 00000000000000..b2f948ab02a613
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_untied_replay.cpp
@@ -0,0 +1,58 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A firstprivate of a non-trivially-copyable type on a replayed untied task
+// that reaches a task scheduling point. The body's copy of it, constructed
+// from the variable as it is when each replay starts (OpenMP 6.0 [14.3]),
+// must last from the task's first part to its last, whichever thread resumes
+// it, and every copy made for an encounter is destroyed by the time the
+// taskgraph construct completes.
+
+#include <cstdio>
+#include <vector>
+
+struct Tracked {
+ static int Live;
+ std::vector<int> V;
+ explicit Tracked(int N) : V(N, 0) { ++Live; }
+ Tracked(const Tracked &O) : V(O.V) { ++Live; }
+ ~Tracked() { --Live; }
+};
+int Tracked::Live = 0;
+
+static size_t Before[4], After[4];
+static int LiveAfter[4];
+
+__attribute__((noinline)) static void run(int R) {
+ Tracked T(R + 1);
+#pragma omp taskgraph graph_id(5)
+ {
+#pragma omp task untied firstprivate(T) shared(Before, After)
+ {
+ Before[R] = T.V.size();
+ T.V.push_back(R);
+#pragma omp taskyield
+ After[R] = T.V.size();
+ }
+ }
+ // T alone.
+ LiveAfter[R] = Tracked::Live;
+}
+
+int main() {
+#pragma omp parallel
+#pragma omp single
+ for (int R = 0; R < 4; ++R)
+ run(R);
+ for (int R = 0; R < 4; ++R)
+ std::printf("run %d: before %zu after %zu live %d\n", R, Before[R],
+ After[R], LiveAfter[R]);
+ return 0;
+}
+
+// CHECK: run 0: before 1 after 2 live 1
+// CHECK-NEXT: run 1: before 2 after 3 live 1
+// CHECK-NEXT: run 2: before 3 after 4 live 1
+// CHECK-NEXT: run 3: before 4 after 5 live 1
diff --git a/openmp/runtime/test/taskgraph/taskgraph_member_replay.cpp b/openmp/runtime/test/taskgraph/taskgraph_member_replay.cpp
new file mode 100644
index 00000000000000..4bb0a2f94879d8
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_member_replay.cpp
@@ -0,0 +1,68 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A taskgraph in a member function, replayed for different objects. A replay
+// starts from the taskgraph construct's enclosing data environment (OpenMP 6.0
+// [14.3]), and there `this` is the object the member function was called for
+// this time. So on every replay:
+//
+// - a task that reaches the object through `this` reaches this call's object;
+// - a firstprivate of a non-static data member, `firstprivate(m)`, whose list
+// item is `this->m`, takes that member's value as it is now -- whatever its
+// type, on a task or on the tasks of a taskloop;
+// - a `saved` one keeps the value it had when the graph was recorded.
+
+#include <cstdio>
+#include <string>
+
+struct S {
+ int M = 0;
+ std::string Str;
+ int Out[8] = {};
+ size_t Len[8] = {};
+ int Saved[8] = {};
+ int Loop[8] = {};
+
+ __attribute__((noinline)) void run(int R) {
+#pragma omp taskgraph graph_id(4)
+ {
+#pragma omp task firstprivate(M, Str) shared(Out, Len)
+ {
+ Out[R] = M;
+ Len[R] = Str.size();
+ M = -1;
+ Str.clear();
+ }
+#pragma omp task firstprivate(saved : M) shared(Saved)
+ Saved[R] = M;
+#pragma omp taskloop firstprivate(M) shared(Loop) num_tasks(1)
+ for (int I = 0; I < 1; ++I)
+ Loop[R] = M;
+ }
+ }
+};
+
+int main() {
+ S A, B;
+#pragma omp parallel
+#pragma omp single
+ for (int R = 0; R < 4; ++R) {
+ S &Obj = (R & 1) ? B : A;
+ Obj.M = 10 * R + 1;
+ Obj.Str.assign(R + 1, 'x');
+ Obj.run(R);
+ }
+ for (int R = 0; R < 4; ++R) {
+ S &Obj = (R & 1) ? B : A;
+ std::printf("run %d: %d %zu %d %d (M %d)\n", R, Obj.Out[R], Obj.Len[R],
+ Obj.Saved[R], Obj.Loop[R], Obj.M);
+ }
+ return 0;
+}
+
+// CHECK: run 0: 1 1 1 1 (M 21)
+// CHECK-NEXT: run 1: 11 2 1 11 (M 31)
+// CHECK-NEXT: run 2: 21 3 1 21 (M 21)
+// CHECK-NEXT: run 3: 31 4 1 31 (M 31)
diff --git a/openmp/runtime/test/taskgraph/taskgraph_private_nontrivial_replay.cpp b/openmp/runtime/test/taskgraph/taskgraph_private_nontrivial_replay.cpp
new file mode 100644
index 00000000000000..35d63b465c593e
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_private_nontrivial_replay.cpp
@@ -0,0 +1,72 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Private copies of a type with a non-trivial constructor and destructor, on
+// a replayed task and on the tasks of a replayed taskloop. Each replay
+// generates its tasks again (OpenMP 6.0 [14.3]), so each gets a new private
+// copy [7.5.3], default-constructed -- or, for a firstprivate,
+// copy-constructed from the variable [7.5.4] -- and destroyed when the task
+// completes. A body's changes to its copy never reach a later replay, and
+// every copy made is destroyed.
+
+#include <atomic>
+#include <cstdio>
+#include <vector>
+
+// The copies of the taskloop's tasks are made and destroyed concurrently.
+struct Tracked {
+ static std::atomic<int> Live;
+ std::vector<int> V;
+ Tracked() { ++Live; }
+ Tracked(const Tracked &O) : V(O.V) { ++Live; }
+ ~Tracked() { --Live; }
+};
+std::atomic<int> Tracked::Live{0};
+
+static size_t TaskSeen[4], LoopSeen[4][2], LoopFirst[4][2];
+
+__attribute__((noinline)) static void run(int R, const Tracked &Init) {
+ Tracked P, F = Init;
+#pragma omp taskgraph graph_id(3)
+ {
+#pragma omp task private(P) shared(TaskSeen)
+ {
+ TaskSeen[R] = P.V.size();
+ P.V.push_back(R);
+ }
+#pragma omp taskloop private(P) firstprivate(F) shared(LoopSeen, LoopFirst) \
+ num_tasks(2)
+ for (int I = 0; I < 2; ++I) {
+ LoopSeen[R][I] = P.V.size();
+ LoopFirst[R][I] = F.V.size();
+ P.V.push_back(I);
+ F.V.push_back(I);
+ }
+ }
+}
+
+int main() {
+#pragma omp parallel
+#pragma omp single
+ {
+ Tracked Init;
+ for (int R = 0; R < 4; ++R) {
+ Init.V.push_back(R);
+ run(R, Init);
+ }
+ }
+ for (int R = 0; R < 4; ++R)
+ std::printf("run %d: task %zu, loop %zu %zu, first %zu %zu\n", R,
+ TaskSeen[R], LoopSeen[R][0], LoopSeen[R][1], LoopFirst[R][0],
+ LoopFirst[R][1]);
+ std::printf("live %d\n", Tracked::Live.load());
+ return 0;
+}
+
+// CHECK: run 0: task 0, loop 0 0, first 1 1
+// CHECK-NEXT: run 1: task 0, loop 0 0, first 2 2
+// CHECK-NEXT: run 2: task 0, loop 0 0, first 3 3
+// CHECK-NEXT: run 3: task 0, loop 0 0, first 4 4
+// CHECK-NEXT: live 0
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture.cpp
new file mode 100644
index 00000000000000..0ea8e4bdadbb38
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture.cpp
@@ -0,0 +1,46 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_mixed_capture(int seed) {
+ int x = seed;
+ int y = seed * 2;
+ int out = -1;
+ int fp = 7;
+
+#pragma omp taskgraph graph_id(401)
+ {
+#pragma omp task replayable(1) shared(x, y, out) firstprivate(fp) \
+ depend(inout : x, y)
+ {
+ x += fp;
+ y += x;
+ out = y + fp;
+ }
+ }
+
+ return out;
+}
+
+int main() {
+ const int first = run_taskgraph_mixed_capture(1);
+ const int second = run_taskgraph_mixed_capture(100);
+
+ if (first != 17 || second != 314) {
+ std::fprintf(stderr,
+ "FAIL lexical mixed capture replay first=%d second=%d "
+ "expected=17/314\n",
+ first, second);
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS lexical mixed capture replay first=%d second=%d\n",
+ first, second);
+ return 0;
+}
+
+// CHECK: PASS lexical mixed capture replay first=17 second=314
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture_taskloop.cpp
new file mode 100644
index 00000000000000..147e768dcc85c2
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture_taskloop.cpp
@@ -0,0 +1,45 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_mixed_capture(int seed) {
+ int x = seed;
+ int y = seed * 2;
+ int fp = 7;
+ int res = 0;
+
+#pragma omp taskgraph graph_id(612)
+ {
+#pragma omp taskloop replayable num_tasks(8) shared(x, y) firstprivate(fp) \
+ reduction(+ : res)
+ for (int i = 0; i < 16; ++i) {
+ res += x + y + fp + i;
+ }
+ }
+
+ return res;
+}
+
+int main() {
+ const int first = run_taskgraph_mixed_capture(1);
+ const int second = run_taskgraph_mixed_capture(100);
+
+ if (first != 280 || second != 5032) {
+ std::fprintf(stderr,
+ "FAIL lexical mixed capture taskloop replay first=%d "
+ "second=%d expected=280/5032\n",
+ first, second);
+ return 1;
+ }
+
+ std::fprintf(
+ stderr, "PASS lexical mixed capture taskloop replay first=%d second=%d\n",
+ first, second);
+ return 0;
+}
+
+// CHECK: PASS lexical mixed capture taskloop replay first=280 second=5032
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type.cpp
new file mode 100644
index 00000000000000..f3a50a27176d4a
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type.cpp
@@ -0,0 +1,60 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+struct Tracker {
+ static int Ctors;
+ static int Dtors;
+
+ int Value;
+
+ explicit Tracker(int V) : Value(V) { ++Ctors; }
+ ~Tracker() { ++Dtors; }
+
+ void bump(int Delta) { Value += Delta; }
+};
+
+int Tracker::Ctors = 0;
+int Tracker::Dtors = 0;
+
+__attribute__((noinline)) static int run_taskgraph_nontrivial(int seed) {
+ Tracker Obj(seed);
+ int out = -1;
+
+#pragma omp taskgraph graph_id(403)
+ {
+#pragma omp task replayable(1) shared(Obj, out)
+ {
+ Obj.bump(11);
+ out = Obj.Value;
+ }
+ }
+
+ return out;
+}
+
+int main() {
+ const int first = run_taskgraph_nontrivial(1);
+ const int second = run_taskgraph_nontrivial(100);
+
+ if (first != 12 || second != 111 || Tracker::Ctors < 2 ||
+ Tracker::Dtors < 2 || Tracker::Ctors != Tracker::Dtors) {
+ std::fprintf(
+ stderr,
+ "FAIL lexical nontrivial replay first=%d second=%d ctors=%d dtors=%d\n",
+ first, second, Tracker::Ctors, Tracker::Dtors);
+ return 1;
+ }
+
+ std::fprintf(
+ stderr,
+ "PASS lexical nontrivial replay first=%d second=%d ctors=%d dtors=%d\n",
+ first, second, Tracker::Ctors, Tracker::Dtors);
+ return 0;
+}
+
+// CHECK: PASS lexical nontrivial replay first=12 second=111
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive.cpp
new file mode 100644
index 00000000000000..6012b6194e56b4
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive.cpp
@@ -0,0 +1,88 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+struct Tracker {
+ static int Ctors;
+ static int Dtors;
+
+ int Value;
+
+ explicit Tracker(int V) : Value(V) { ++Ctors; }
+ ~Tracker() { ++Dtors; }
+
+ void bump(int Delta) { Value += Delta; }
+};
+
+int Tracker::Ctors = 0;
+int Tracker::Dtors = 0;
+
+__attribute__((noinline)) static int expected_recursive(int depth, int seed,
+ int run_tag) {
+ int local = seed + (depth + 1) * 5 + run_tag;
+ if (depth == 0)
+ return local;
+ return local + expected_recursive(depth - 1, seed + 9, run_tag);
+}
+
+__attribute__((noinline)) static int
+run_recursive_nontrivial(int depth, int seed, int run_tag) {
+ Tracker Obj(seed);
+ int out = -1;
+
+ int gid = 500 + depth;
+#pragma omp taskgraph graph_id(gid)
+ {
+#pragma omp task replayable(1) shared(Obj, out, depth, run_tag)
+ {
+ Obj.bump((depth + 1) * 5 + run_tag);
+ out = Obj.Value;
+ }
+ }
+
+ if (depth == 0)
+ return out;
+ return out + run_recursive_nontrivial(depth - 1, seed + 9, run_tag);
+}
+
+int main() {
+ const int depth = 3;
+ int total_actual = 0;
+ int total_expected = 0;
+
+ for (int run = 0; run < 3; ++run) {
+ const int seed = 100 * run + 1;
+ const int actual = run_recursive_nontrivial(depth, seed, run);
+ const int expected = expected_recursive(depth, seed, run);
+
+ if (actual != expected) {
+ std::fprintf(stderr,
+ "FAIL recursive nontrivial run=%d actual=%d expected=%d\n",
+ run, actual, expected);
+ return 1;
+ }
+
+ total_actual += actual;
+ total_expected += expected;
+ }
+
+ if (Tracker::Ctors != Tracker::Dtors || Tracker::Ctors < 12) {
+ std::fprintf(stderr,
+ "FAIL recursive nontrivial lifetime ctors=%d dtors=%d "
+ "total=%d expected=%d\n",
+ Tracker::Ctors, Tracker::Dtors, total_actual, total_expected);
+ return 1;
+ }
+
+ std::fprintf(
+ stderr,
+ "PASS recursive nontrivial total=%d expected=%d ctors=%d dtors=%d\n",
+ total_actual, total_expected, Tracker::Ctors, Tracker::Dtors);
+ return 0;
+}
+
+// CHECK: PASS recursive nontrivial total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive_taskloop.cpp
new file mode 100644
index 00000000000000..ab23adf6ce4775
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive_taskloop.cpp
@@ -0,0 +1,89 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+struct Tracker {
+ static int Ctors;
+ static int Dtors;
+
+ int Value;
+
+ explicit Tracker(int V) : Value(V) { ++Ctors; }
+ ~Tracker() { ++Dtors; }
+
+ void bump(int Delta) { Value += Delta; }
+};
+
+int Tracker::Ctors = 0;
+int Tracker::Dtors = 0;
+
+__attribute__((noinline)) static int expected_recursive(int depth, int seed,
+ int run_tag) {
+ int local = 16 * (seed + (depth + 1) * 5 + run_tag) + 120;
+ if (depth == 0)
+ return local;
+ return local + expected_recursive(depth - 1, seed + 9, run_tag);
+}
+
+__attribute__((noinline)) static int
+run_recursive_nontrivial(int depth, int seed, int run_tag) {
+ Tracker Obj(seed);
+ int res = 0;
+
+ int gid = 620 + depth;
+#pragma omp taskgraph graph_id(gid)
+ {
+#pragma omp taskloop replayable num_tasks(8) shared(Obj, depth, run_tag) \
+ reduction(+ : res)
+ for (int i = 0; i < 16; ++i) {
+ res += Obj.Value + (depth + 1) * 5 + run_tag + i;
+ }
+ }
+
+ if (depth == 0)
+ return res;
+ return res + run_recursive_nontrivial(depth - 1, seed + 9, run_tag);
+}
+
+int main() {
+ const int depth = 3;
+ int total_actual = 0;
+ int total_expected = 0;
+
+ for (int run = 0; run < 3; ++run) {
+ const int seed = 100 * run + 1;
+ const int actual = run_recursive_nontrivial(depth, seed, run);
+ const int expected = expected_recursive(depth, seed, run);
+
+ if (actual != expected) {
+ std::fprintf(
+ stderr,
+ "FAIL recursive nontrivial taskloop run=%d actual=%d expected=%d\n",
+ run, actual, expected);
+ return 1;
+ }
+
+ total_actual += actual;
+ total_expected += expected;
+ }
+
+ if (Tracker::Ctors != Tracker::Dtors || Tracker::Ctors < 12) {
+ std::fprintf(stderr,
+ "FAIL recursive nontrivial taskloop lifetime ctors=%d "
+ "dtors=%d total=%d expected=%d\n",
+ Tracker::Ctors, Tracker::Dtors, total_actual, total_expected);
+ return 1;
+ }
+
+ std::fprintf(stderr,
+ "PASS recursive nontrivial taskloop total=%d expected=%d "
+ "ctors=%d dtors=%d\n",
+ total_actual, total_expected, Tracker::Ctors, Tracker::Dtors);
+ return 0;
+}
+
+// CHECK: PASS recursive nontrivial taskloop total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_taskloop.cpp
new file mode 100644
index 00000000000000..9d4d3dd15b7d2c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_taskloop.cpp
@@ -0,0 +1,59 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+struct Tracker {
+ static int Ctors;
+ static int Dtors;
+
+ int Value;
+
+ explicit Tracker(int V) : Value(V) { ++Ctors; }
+ ~Tracker() { ++Dtors; }
+
+ void bump(int Delta) { Value += Delta; }
+};
+
+int Tracker::Ctors = 0;
+int Tracker::Dtors = 0;
+
+__attribute__((noinline)) static int run_taskgraph_nontrivial(int seed) {
+ Tracker Obj(seed);
+ int res = 0;
+
+#pragma omp taskgraph graph_id(614)
+ {
+#pragma omp taskloop replayable num_tasks(8) shared(Obj) reduction(+ : res)
+ for (int i = 0; i < 16; ++i) {
+ res += Obj.Value + i;
+ }
+ }
+
+ return res;
+}
+
+int main() {
+ const int first = run_taskgraph_nontrivial(1);
+ const int second = run_taskgraph_nontrivial(100);
+
+ if (first != 136 || second != 1720 || Tracker::Ctors < 2 ||
+ Tracker::Dtors < 2 || Tracker::Ctors != Tracker::Dtors) {
+ std::fprintf(stderr,
+ "FAIL lexical nontrivial taskloop replay first=%d second=%d "
+ "ctors=%d dtors=%d\n",
+ first, second, Tracker::Ctors, Tracker::Dtors);
+ return 1;
+ }
+
+ std::fprintf(stderr,
+ "PASS lexical nontrivial taskloop replay first=%d second=%d "
+ "ctors=%d dtors=%d\n",
+ first, second, Tracker::Ctors, Tracker::Dtors);
+ return 0;
+}
+
+// CHECK: PASS lexical nontrivial taskloop replay first=136 second=1720
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer.cpp
new file mode 100644
index 00000000000000..7690aff37853cf
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer.cpp
@@ -0,0 +1,44 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_pointer_shared(int seed) {
+ int value = seed;
+ int *ptr = &value;
+ int out = -1;
+
+#pragma omp taskgraph graph_id(402)
+ {
+#pragma omp task replayable(1) shared(ptr, out) depend(inout : value)
+ {
+ *ptr += 3;
+ out = *ptr;
+ }
+ }
+
+ return out;
+}
+
+int main() {
+ const int first = run_taskgraph_pointer_shared(1);
+ const int second = run_taskgraph_pointer_shared(100);
+
+ if (first != 4 || second != 103) {
+ std::fprintf(stderr,
+ "FAIL lexical pointer shared replay first=%d second=%d "
+ "expected=4/103\n",
+ first, second);
+ return 1;
+ }
+
+ std::fprintf(stderr,
+ "PASS lexical pointer shared replay first=%d second=%d\n", first,
+ second);
+ return 0;
+}
+
+// CHECK: PASS lexical pointer shared replay first=4 second=103
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid.cpp
new file mode 100644
index 00000000000000..142a5941f97ae9
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid.cpp
@@ -0,0 +1,76 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdint>
+#include <cstdio>
+
+__attribute__((noinline)) static int expected_recursive(int depth, int seed,
+ int run_tag) {
+ int value = seed;
+ value += (depth + 1) * 3 + run_tag;
+ if (depth == 0)
+ return value;
+ return value + expected_recursive(depth - 1, seed + 7, run_tag);
+}
+
+__attribute__((noinline)) static int run_recursive_frameid(int depth, int seed,
+ int run_tag) {
+ int value = seed;
+ int *ptr = &value;
+ int *&ptr_ref = ptr;
+ int out = -1;
+
+ // Typically, if captured pointers refer to locations on the stack, that
+ // would not be safe for taskgraph record/replay because we in general we
+ // cannot rewrite such pointers to point to the current (live) stack frame.
+ //
+ // This is one possible way around that though: we keep a taskgraph record
+ // per stack-depth, each of which may refer to the local stack frame.
+ //
+ // I probably wouldn't recommend use of this technique in production code.
+ uintptr_t frame_gid = reinterpret_cast<uintptr_t>(__builtin_frame_address(0));
+
+#pragma omp taskgraph graph_id(frame_gid)
+ {
+#pragma omp task shared(ptr_ref, out, depth, run_tag) depend(inout : value)
+ {
+ *ptr_ref += (depth + 1) * 3 + run_tag;
+ out = *ptr_ref;
+ }
+ }
+
+ if (depth == 0)
+ return out;
+ return out + run_recursive_frameid(depth - 1, seed + 7, run_tag);
+}
+
+int main() {
+ const int depth = 3;
+ int actual_sum = 0;
+ int expected_sum = 0;
+
+ for (int run = 0; run < 3; ++run) {
+ int seed = 100 * run + 1;
+ int actual = run_recursive_frameid(depth, seed, run);
+ int expected = expected_recursive(depth, seed, run);
+ if (actual != expected) {
+ std::fprintf(
+ stderr,
+ "FAIL recursive pointer frameid run=%d actual=%d expected=%d\n", run,
+ actual, expected);
+ return 1;
+ }
+ actual_sum += actual;
+ expected_sum += expected;
+ }
+
+ std::fprintf(stderr,
+ "PASS recursive pointer frameid runs=3 total=%d expected=%d\n",
+ actual_sum, expected_sum);
+ return 0;
+}
+
+// CHECK: PASS recursive pointer frameid runs=3 total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid_taskloop.cpp
new file mode 100644
index 00000000000000..31db8cac9b28fd
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid_taskloop.cpp
@@ -0,0 +1,95 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+#include <cstdint>
+
+__attribute__((noinline)) static int expected_recursive(int depth, int seed,
+ int run_tag) {
+ // Mirror run_recursive_frameid: the taskloop adds, over i in [0, 16),
+ // delta = (depth + 1) * 3 + run_tag + i both to 'value' (through the shared
+ // pointer) and to the reduction 'sum_delta'.
+ int sum_delta = 16 * ((depth + 1) * 3 + run_tag) + 120;
+ int value = seed + sum_delta;
+ int local = value * 17 + sum_delta;
+ if (depth == 0)
+ return local;
+ return local + expected_recursive(depth - 1, seed + 7, run_tag);
+}
+
+__attribute__((noinline)) static int run_recursive_frameid(int depth, int seed,
+ int run_tag) {
+ int value = seed;
+ int *ptr = &value;
+ int *&ptr_ref = ptr;
+ int sum_delta = 0;
+ uintptr_t frame_gid = reinterpret_cast<uintptr_t>(__builtin_frame_address(0));
+
+ // Typically, if captured pointers refer to locations on the stack, that
+ // would not be safe for taskgraph record/replay because we in general we
+ // cannot rewrite such pointers to point to the current (live) stack frame.
+ //
+ // This is one possible way around that though: we keep a taskgraph record
+ // per stack-depth, each of which may refer to the local stack frame.
+ //
+ // I probably wouldn't recommend use of this technique in production code.
+#pragma omp taskgraph graph_id(frame_gid)
+ {
+#pragma omp taskloop replayable num_tasks(8) shared(ptr_ref, depth, run_tag) \
+ reduction(+ : sum_delta)
+ for (int i = 0; i < 16; ++i) {
+ int delta = (depth + 1) * 3 + run_tag + i;
+ __atomic_fetch_add(ptr_ref, delta, __ATOMIC_RELAXED);
+ sum_delta += delta;
+ }
+ }
+
+ int local = value * 17 + sum_delta;
+
+ if (depth == 0)
+ return local;
+ return local + run_recursive_frameid(depth - 1, seed + 7, run_tag);
+}
+
+int main() {
+ const int depth = 3;
+ int actual_sum = 0;
+ int expected_sum = 0;
+ int failures = 0;
+
+#pragma omp parallel
+ {
+#pragma omp single
+ {
+ for (int run = 0; run < 3; ++run) {
+ int seed = 100 * run + 1;
+ int actual = run_recursive_frameid(depth, seed, run);
+ int expected = expected_recursive(depth, seed, run);
+ if (actual != expected) {
+ std::fprintf(
+ stderr,
+ "FAIL recursive pointer taskloop replay run=%d actual=%d "
+ "expected=%d\n",
+ run, actual, expected);
+ ++failures;
+ }
+ actual_sum += actual;
+ expected_sum += expected;
+ }
+ }
+ }
+
+ if (failures != 0)
+ return 1;
+
+ std::fprintf(stderr,
+ "PASS recursive pointer taskloop replay runs=3 total=%d "
+ "expected=%d\n",
+ actual_sum, expected_sum);
+ return 0;
+}
+
+// CHECK: PASS recursive pointer taskloop replay
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_taskloop.cpp
new file mode 100644
index 00000000000000..ac5471505e5395
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_taskloop.cpp
@@ -0,0 +1,44 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_pointer_shared(int seed) {
+ int value = seed;
+ int *ptr = &value;
+ int res = 0;
+
+#pragma omp taskgraph graph_id(613)
+ {
+#pragma omp taskloop replayable num_tasks(8) shared(ptr) reduction(+ : res)
+ for (int i = 0; i < 16; ++i) {
+ res += *ptr + i;
+ }
+ }
+
+ return res;
+}
+
+int main() {
+ const int first = run_taskgraph_pointer_shared(1);
+ const int second = run_taskgraph_pointer_shared(100);
+
+ if (first != 136 || second != 1720) {
+ std::fprintf(stderr,
+ "FAIL lexical pointer shared taskloop replay first=%d "
+ "second=%d expected=136/1720\n",
+ first, second);
+ return 1;
+ }
+
+ std::fprintf(
+ stderr,
+ "PASS lexical pointer shared taskloop replay first=%d second=%d\n", first,
+ second);
+ return 0;
+}
+
+// CHECK: PASS lexical pointer shared taskloop replay first=136 second=1720
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive.cpp
new file mode 100644
index 00000000000000..4d0d7b5613108a
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive.cpp
@@ -0,0 +1,46 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_recursive(int depth,
+ int seed) {
+ int x = seed;
+ int out = -1;
+
+#pragma omp taskgraph graph_id(450)
+ {
+#pragma omp task replayable(1) shared(x, out, depth) depend(inout : x)
+ {
+ x += depth + 1;
+ out = x;
+ }
+ }
+
+ if (depth == 0)
+ return out;
+
+ return out + run_taskgraph_recursive(depth - 1, seed + 10);
+}
+
+int main() {
+ const int first = run_taskgraph_recursive(3, 1);
+ const int second = run_taskgraph_recursive(3, 100);
+
+ if (first != 74 || second != 470) {
+ std::fprintf(
+ stderr,
+ "FAIL lexical recursive replay first=%d second=%d expected=74/470\n",
+ first, second);
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS lexical recursive replay first=%d second=%d\n",
+ first, second);
+ return 0;
+}
+
+// CHECK: PASS lexical recursive replay first=74 second=470
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive_taskloop.cpp
new file mode 100644
index 00000000000000..8d3df091be741f
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive_taskloop.cpp
@@ -0,0 +1,72 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int expected_recursive(int depth, int seed) {
+ int x = seed;
+ int sum_delta = 0;
+
+ for (int i = 0; i < 16; ++i) {
+ sum_delta += depth + i + 1;
+ }
+
+ x += sum_delta;
+ int local = x * 17 + sum_delta;
+
+ if (depth == 0)
+ return local;
+
+ return local + expected_recursive(depth - 1, seed + 10);
+}
+
+__attribute__((noinline)) static int run_taskgraph_recursive(int depth,
+ int seed) {
+ int x = seed;
+ int *ptr = &x;
+ int sum_delta = 0;
+ int gid = 615;
+
+#pragma omp taskgraph graph_id(gid)
+ {
+#pragma omp taskloop replayable num_tasks(8) shared(ptr, depth) \
+ reduction(+ : sum_delta)
+ for (int i = 0; i < 16; ++i) {
+ int delta = depth + i + 1;
+ __atomic_fetch_add(ptr, delta, __ATOMIC_RELAXED);
+ sum_delta += delta;
+ }
+ }
+
+ int local = x * 17 + sum_delta;
+
+ if (depth == 0)
+ return local;
+
+ return local + run_taskgraph_recursive(depth - 1, seed + 10);
+}
+
+int main() {
+ const int first = run_taskgraph_recursive(3, 1);
+ const int second = run_taskgraph_recursive(3, 100);
+ const int expected_first = expected_recursive(3, 1);
+ const int expected_second = expected_recursive(3, 100);
+
+ if (first == expected_first && second == expected_second) {
+ std::fprintf(stderr,
+ "PASS lexical recursive taskloop replay first=%d second=%d\n",
+ first, second);
+ return 0;
+ }
+
+ std::fprintf(stderr,
+ "FAIL lexical recursive taskloop replay first=%d second=%d "
+ "expected=%d/%d\n",
+ first, second, expected_first, expected_second);
+ return 1;
+}
+
+// CHECK: PASS lexical recursive taskloop replay
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_this.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_this.cpp
new file mode 100644
index 00000000000000..d407637dc096db
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_this.cpp
@@ -0,0 +1,34 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A taskgraph in a member function, replayed for different objects. A task
+// that reaches the object through `this` must reach the object the function
+// was called for this time, which the taskgraph's record holds by value.
+
+#include <cstdio>
+
+struct Counter {
+ int Hits = 0;
+ __attribute__((noinline)) void run() {
+#pragma omp taskgraph graph_id(11)
+ {
+#pragma omp task
+ ++Hits;
+ }
+ }
+};
+
+int main() {
+ Counter A, B;
+#pragma omp parallel
+#pragma omp single
+ for (int R = 0; R < 6; ++R)
+ ((R & 1) ? B : A).run();
+ std::printf("A %d B %d\n", A.Hits, B.Hits);
+ return 0;
+}
+
+// CHECK: A 3 B 3
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works.cpp
new file mode 100644
index 00000000000000..82be9c82d3e7c9
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works.cpp
@@ -0,0 +1,42 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_lexical(int seed) {
+ int x = seed;
+ int out = -1;
+
+#pragma omp taskgraph graph_id(311)
+ {
+#pragma omp task replayable(1) shared(x, out) depend(inout : x)
+ {
+ x += 5;
+ out = x;
+ }
+ }
+
+ return out;
+}
+
+int main() {
+ const int first = run_taskgraph_lexical(1);
+ const int second = run_taskgraph_lexical(100);
+
+ if (first != 6 || second != 105) {
+ std::fprintf(
+ stderr,
+ "FAIL lexical shared replay first=%d second=%d expected=6/105\n", first,
+ second);
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS lexical shared replay first=%d second=%d\n", first,
+ second);
+ return 0;
+}
+
+// CHECK: PASS lexical shared replay first=6 second=105
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works_taskloop.cpp
new file mode 100644
index 00000000000000..beaaedfea77f3d
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works_taskloop.cpp
@@ -0,0 +1,42 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_lexical(int seed) {
+ int x = seed;
+ int res = 0;
+
+#pragma omp taskgraph graph_id(611)
+ {
+#pragma omp taskloop replayable num_tasks(8) shared(x) reduction(+ : res)
+ for (int i = 0; i < 16; ++i) {
+ res += x + i;
+ }
+ }
+
+ return res;
+}
+
+int main() {
+ const int first = run_taskgraph_lexical(1);
+ const int second = run_taskgraph_lexical(100);
+
+ if (first != 136 || second != 1720) {
+ std::fprintf(stderr,
+ "FAIL lexical shared taskloop replay first=%d second=%d "
+ "expected=136/1720\n",
+ first, second);
+ return 1;
+ }
+
+ std::fprintf(stderr,
+ "PASS lexical shared taskloop replay first=%d second=%d\n",
+ first, second);
+ return 0;
+}
+
+// CHECK: PASS lexical shared taskloop replay first=136 second=1720
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1.cpp
new file mode 100644
index 00000000000000..1d88fccfe4c8b4
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1.cpp
@@ -0,0 +1,52 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %not --crash %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+// This seems like it could work in principle, but in general we don't know
+// where the targets of the referenced variables are when the task is replayed.
+__attribute__((noinline)) static void emit_nonlexical_task(int &x, int &out) {
+#pragma omp task replayable(1) shared(x, out) depend(inout : x)
+ {
+ x += 5;
+ out = x;
+ }
+}
+
+__attribute__((noinline)) static int run_taskgraph_nonlexical(int seed) {
+ int x = seed;
+ int out = -1;
+
+#pragma omp taskgraph graph_id(312)
+ {
+ emit_nonlexical_task(x, out);
+ }
+
+ return out;
+}
+
+int main() {
+ const int recorded = run_taskgraph_nonlexical(1);
+ const int replayed = run_taskgraph_nonlexical(100);
+
+ // The "non-lexical" replayable task is emitted in a helper function outside
+ // the taskgraph lexical scope. We expect this to raise a runtime error.
+ if (recorded == replayed) {
+ std::fprintf(
+ stderr,
+ "UNEXPECTED SUCCESS nonlexical replay recorded=%d replayed=%d\n",
+ recorded, replayed);
+ return 0;
+ }
+
+ std::fprintf(stderr,
+ "EXPECTED FAILURE nonlexical replay recorded=%d replayed=%d\n",
+ recorded, replayed);
+ return 1;
+}
+
+// clang-format off
+// CHECK: OMP: Error #{{[0-9]+}}: Cannot locate captured shared variable reference for taskgraph replay
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1_taskloop.cpp
new file mode 100644
index 00000000000000..3f873c2a5f2cfa
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1_taskloop.cpp
@@ -0,0 +1,51 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %not --crash %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int emit_nonlexical_taskloop(int &x) {
+ int sum = 0;
+#pragma omp taskloop replayable num_tasks(8) shared(x) reduction(+ : sum)
+ for (int i = 0; i < 16; ++i) {
+ sum += x + i;
+ }
+ x += 5;
+ return sum;
+}
+
+__attribute__((noinline)) static int run_taskgraph_nonlexical(int seed) {
+ int x = seed;
+ int out = -1;
+
+#pragma omp taskgraph graph_id(631)
+ {
+ out = emit_nonlexical_taskloop(x);
+ }
+
+ return out;
+}
+
+int main() {
+ const int recorded = run_taskgraph_nonlexical(1);
+ const int replayed = run_taskgraph_nonlexical(100);
+
+ if (recorded == replayed) {
+ std::fprintf(stderr,
+ "UNEXPECTED SUCCESS nonlexical taskloop replay recorded=%d "
+ "replayed=%d\n",
+ recorded, replayed);
+ return 0;
+ }
+
+ std::fprintf(
+ stderr,
+ "EXPECTED FAILURE nonlexical taskloop replay recorded=%d replayed=%d\n",
+ recorded, replayed);
+ return 1;
+}
+
+// clang-format off
+// CHECK: OMP: Error #{{[0-9]+}}: Cannot locate captured shared variable reference for taskgraph replay
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_in_task.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_in_task.cpp
new file mode 100644
index 00000000000000..43ee1bcf5739f5
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_in_task.cpp
@@ -0,0 +1,53 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A 'replayable' task generated in a function called from the body of a task
+// of the taskgraph region. The body of a task generated in the region is not
+// part of the region (OpenMP 6.0 [2], region), so the inner task is not a
+// replayable construct of it and is not recorded, though it shares variables
+// of a frame that does not exist when the graph is replayed: each replay runs
+// the outer task's body again, which generates the inner task afresh, sharing
+// that replay's variables.
+
+#include <cstdio>
+
+__attribute__((noinline)) static int emit_nonlexical_task(int seed) {
+ int x = seed;
+ int out = -1;
+
+#pragma omp task replayable(1) shared(x, out) depend(inout : x)
+ {
+ x += 5;
+ out = x;
+ }
+#pragma omp taskwait
+
+ return out;
+}
+
+__attribute__((noinline)) static int run_taskgraph_nonlexical(int seed) {
+ int out;
+
+#pragma omp taskgraph graph_id(312)
+ {
+#pragma omp task shared(out) firstprivate(seed)
+ {
+ out = emit_nonlexical_task(seed);
+ }
+ }
+
+ return out;
+}
+
+int main() {
+ const int Seeds[] = {1, 100, 1000};
+ for (int seed : Seeds)
+ std::printf("seed %d: %d\n", seed, run_taskgraph_nonlexical(seed));
+ return 0;
+}
+
+// CHECK: seed 1: 6
+// CHECK-NEXT: seed 100: 105
+// CHECK-NEXT: seed 1000: 1005
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_in_task_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_in_task_taskloop.cpp
new file mode 100644
index 00000000000000..9a90ce73b2ecc2
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_in_task_taskloop.cpp
@@ -0,0 +1,48 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// As taskgraph_replayable_nonlexical_shared_in_task.cpp, for a 'replayable'
+// taskloop: it is not a replayable construct of the region, so each replay
+// runs it afresh, sharing that replay's variables.
+
+#include <cstdio>
+
+__attribute__((noinline)) static int emit_nonlexical_taskloop(int seed) {
+ int x = seed;
+ int sum = 0;
+
+#pragma omp taskloop replayable num_tasks(8) shared(x) reduction(+ : sum)
+ for (int i = 0; i < 16; ++i) {
+ sum += x + i;
+ }
+
+ return sum;
+}
+
+__attribute__((noinline)) static int run_taskgraph_nonlexical(int seed) {
+ int out;
+
+#pragma omp taskgraph graph_id(632)
+ {
+#pragma omp task shared(out) firstprivate(seed)
+ {
+ out = emit_nonlexical_taskloop(seed);
+ }
+ }
+
+ return out;
+}
+
+int main() {
+ // 16 * seed + (0 + ... + 15).
+ const int Seeds[] = {1, 100, 1000};
+ for (int seed : Seeds)
+ std::printf("seed %d: %d\n", seed, run_taskgraph_nonlexical(seed));
+ return 0;
+}
+
+// CHECK: seed 1: 136
+// CHECK-NEXT: seed 100: 1720
+// CHECK-NEXT: seed 1000: 16120
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp
new file mode 100644
index 00000000000000..aa4eec17ad0919
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp
@@ -0,0 +1,118 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+static volatile int ReplayResult = -1;
+
+struct Payload {
+ int values[6];
+ int bias;
+};
+
+__attribute__((noinline)) static int evaluate_payload(const Payload &payload,
+ int seed) {
+ return seed * payload.values[0] - payload.values[1] +
+ payload.values[2] * payload.values[3] - payload.bias +
+ payload.values[4] - payload.values[5];
+}
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+ volatile int scratch[4096];
+
+ for (int i = 0; i < 4096; ++i)
+ scratch[i] = base + i;
+
+ StackSink += scratch[base & 63];
+}
+
+// Intended future usage of firstprivate(saved: ...): this replayable task is
+// not lexically nested within the taskgraph directive. It is created from a
+// helper function called inside the taskgraph region, and needs closure-like
+// capture of that helper's stack locals.
+__attribute__((noinline)) static void emit_replayable_task(int seed) {
+ Payload payload{
+ {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+ seed * 17 + 19};
+
+#pragma omp task replayable(1) firstprivate(saved : payload, seed) \
+ shared(ReplayResult)
+ {
+ ReplayResult = evaluate_payload(payload, seed);
+ }
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+ ReplayResult = -1;
+
+#pragma omp taskgraph graph_id(93)
+ {
+ emit_replayable_task(seed);
+ }
+
+ return ReplayResult;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+ volatile int padding[128];
+
+ for (int i = 0; i < 128; ++i)
+ padding[i] = seed + depth + i;
+
+ StackSink += padding[(seed + depth) & 127];
+
+ if (depth == 0)
+ return run_taskgraph(seed);
+ return call_with_depth(seed, depth - 1);
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+ Payload payload{
+ {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+ seed * 17 + 19};
+ return evaluate_payload(payload, seed);
+}
+
+int main() {
+ constexpr int NumCalls = 4;
+ constexpr int Seeds[NumCalls] = {3, 17, 29, 41};
+ constexpr int Depths[NumCalls] = {0, 3, 1, 5};
+
+ int recorded = -1;
+ bool failed = false;
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ recorded = call_with_depth(Seeds[0], Depths[0]);
+ if (recorded != expected_result(Seeds[0])) {
+ std::fprintf(stderr, "FAIL initial record got=%d expected=%d\n",
+ recorded, expected_result(Seeds[0]));
+ failed = true;
+ }
+
+ for (int i = 1; i < NumCalls; ++i) {
+ clobber_stack(Seeds[i] * 1000);
+ const int replayed = call_with_depth(Seeds[i], Depths[i]);
+ if (replayed != recorded) {
+ std::fprintf(stderr,
+ "FAIL replay %d depth=%d seed=%d got=%d expected=%d\n",
+ i, Depths[i], Seeds[i], replayed, recorded);
+ failed = true;
+ }
+ }
+ }
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr, "PASS replayable saved stack result=%d\n", recorded);
+ return 0;
+}
+
+// CHECK: PASS replayable saved stack result=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth_taskloop.cpp
new file mode 100644
index 00000000000000..5a5b8352f49257
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth_taskloop.cpp
@@ -0,0 +1,130 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+// Aggregated across the taskloop body via #pragma omp atomic. Lives in
+// static storage so the address never moves across replays (this test is
+// about firstprivate(saved:), not reductions: reductions on replayable
+// taskgraphs are tracked separately).
+static volatile int Aggregate = 0;
+
+struct Payload {
+ int values[6];
+ int bias;
+};
+
+__attribute__((noinline)) static int evaluate_payload(const Payload &payload,
+ int seed) {
+ return seed * payload.values[0] - payload.values[1] +
+ payload.values[2] * payload.values[3] - payload.bias +
+ payload.values[4] - payload.values[5];
+}
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+ volatile int scratch[4096];
+
+ for (int i = 0; i < 4096; ++i)
+ scratch[i] = base + i;
+
+ StackSink += scratch[base & 63];
+}
+
+// The taskloop directive lives in a helper function called from inside the
+// taskgraph region, so the loop's firstprivate(saved:) captures originate
+// in a non-lexical surrounding scope. The taskloop relocation helper has
+// nothing to refresh at replay since the saved snapshots are sourced from
+// '.kmp_privates.t', not the shareds slots.
+__attribute__((noinline)) static void emit_replayable_taskloop(int seed) {
+ Payload payload{
+ {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+ seed * 17 + 19};
+
+#pragma omp taskloop replayable num_tasks(8) firstprivate(saved : payload, seed)
+ for (int i = 0; i < 16; ++i) {
+ int contribution = evaluate_payload(payload, seed + i);
+#pragma omp atomic
+ Aggregate += contribution;
+ }
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+ Aggregate = 0;
+
+#pragma omp taskgraph graph_id(633)
+ {
+ emit_replayable_taskloop(seed);
+ }
+
+ return Aggregate;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+ volatile int padding[128];
+
+ for (int i = 0; i < 128; ++i)
+ padding[i] = seed + depth + i;
+
+ StackSink += padding[(seed + depth) & 127];
+
+ if (depth == 0)
+ return run_taskgraph(seed);
+ return call_with_depth(seed, depth - 1);
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+ Payload payload{
+ {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+ seed * 17 + 19};
+ int sum = 0;
+ for (int i = 0; i < 16; ++i)
+ sum += evaluate_payload(payload, seed + i);
+ return sum;
+}
+
+int main() {
+ constexpr int NumCalls = 4;
+ constexpr int Seeds[NumCalls] = {3, 17, 29, 41};
+ constexpr int Depths[NumCalls] = {0, 3, 1, 5};
+
+ int recorded = -1;
+ bool failed = false;
+
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ {
+ recorded = call_with_depth(Seeds[0], Depths[0]);
+ if (recorded != expected_result(Seeds[0])) {
+ std::fprintf(stderr,
+ "FAIL initial taskloop record got=%d expected=%d\n",
+ recorded, expected_result(Seeds[0]));
+ failed = true;
+ }
+
+ for (int i = 1; i < NumCalls; ++i) {
+ clobber_stack(Seeds[i] * 1000);
+ const int replayed = call_with_depth(Seeds[i], Depths[i]);
+ if (replayed != recorded) {
+ std::fprintf(
+ stderr,
+ "FAIL taskloop replay %d depth=%d seed=%d got=%d expected=%d\n",
+ i, Depths[i], Seeds[i], replayed, recorded);
+ failed = true;
+ }
+ }
+ }
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr, "PASS replayable taskloop saved stack result=%d\n",
+ recorded);
+ return 0;
+}
+
+// CHECK: PASS replayable taskloop saved stack result=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp b/openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp
new file mode 100644
index 00000000000000..ab93aac8b41f5c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp
@@ -0,0 +1,104 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// This test exercises a non-lexically-nested replayable task that captures
+// 'payload' and 'seed' as 'firstprivate' (without the OpenMP 6.0 'saved'
+// modifier). Both 'payload' and 'seed' live in run_taskgraph's frame,
+// which is the enclosing data environment of the taskgraph construct, so
+// per OpenMP 6.0 [14.3] each replay initializes the firstprivates from
+// the variables' current values rather than from the saved data
+// environment. The replay is entered at a different call-stack depth
+// than the recording, so the refresh has to read the current frame, not
+// the one the task was recorded in.
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+
+struct Payload {
+ int values[6];
+ int bias;
+};
+
+__attribute__((noinline)) static int evaluate_payload(const Payload &payload,
+ int seed) {
+ return seed * payload.values[0] - payload.values[1] +
+ payload.values[2] * payload.values[3] - payload.bias +
+ payload.values[4] - payload.values[5];
+}
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+ volatile int scratch[4096];
+
+ for (int i = 0; i < 4096; ++i)
+ scratch[i] = base + i;
+
+ StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+ Payload payload{
+ {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+ seed * 17 + 19};
+ int result = -1;
+
+#pragma omp taskgraph graph_id(92)
+ {
+#pragma omp task firstprivate(payload, seed) shared(result)
+ {
+ result = evaluate_payload(payload, seed);
+ }
+ }
+
+ return result;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+ volatile int padding[128];
+
+ for (int i = 0; i < 128; ++i)
+ padding[i] = seed + depth + i;
+
+ StackSink += padding[(seed + depth) & 127];
+
+ if (depth == 0)
+ return run_taskgraph(seed);
+ return call_with_depth(seed, depth - 1);
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+ Payload payload{
+ {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+ seed * 17 + 19};
+ return evaluate_payload(payload, seed);
+}
+
+int main() {
+ constexpr int RecordSeed = 3;
+ constexpr int ReplaySeed = 17;
+
+ const int recorded = call_with_depth(RecordSeed, 0);
+ if (recorded != expected_result(RecordSeed)) {
+ std::fprintf(stderr, "FAIL initial record got=%d expected=%d\n", recorded,
+ expected_result(RecordSeed));
+ return 1;
+ }
+
+ clobber_stack(ReplaySeed * 1000);
+ const int replayed = call_with_depth(ReplaySeed, 3);
+ if (replayed != expected_result(ReplaySeed)) {
+ std::fprintf(
+ stderr, "BUG shared stack replay depth=%d seed=%d got=%d expected=%d\n",
+ 3, ReplaySeed, replayed, expected_result(ReplaySeed));
+ return 1;
+ }
+
+ std::fprintf(stderr, "PASS shared stack record=%d replay=%d\n", recorded,
+ replayed);
+ return 0;
+}
+
+// CHECK: PASS shared stack record=14 replay=504
diff --git a/openmp/runtime/test/taskgraph/taskgraph_untied_taskyield.cpp b/openmp/runtime/test/taskgraph/taskgraph_untied_taskyield.cpp
new file mode 100644
index 00000000000000..6ebee922477844
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_untied_taskyield.cpp
@@ -0,0 +1,131 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Nothing in the spec stops a recorded task being untied or reaching a task
+// scheduling point: untied only says any thread in the binding thread set may
+// resume the region after suspension, and the only restriction 6.0 adds to the
+// taskgraph construct is that antecedent tasks across constructs agree on being
+// replayable. So both have to work on replay.
+//
+// They do, and what makes them work is worth pinning down, because it is not
+// obvious and it is easy to break. A recorded untied task really does run in
+// several parts, so __kmp_task_finish is reached more than once for the same
+// node. The untied part counter is checked *before* the taskgraph successor
+// hook there, so a node releases its successors -- and decrements its taskgroup
+// count -- only on its final part. Release them on an earlier part instead and
+// the graph fires successors early and repeatedly; skip the release and the
+// graph stalls.
+//
+// That in turn is what lets __kmp_omp_tg_task zero td_untied_count at the start
+// of every replay: map_lock serialises replays, and because the count only
+// drops on the last part, the implicit taskgroup wait has drained every part of
+// every node before the next replay begins. Genuinely concurrent replay, or
+// honouring nogroup on replay, would zero a live part counter and strand the
+// successors of a half-finished node.
+
+#include <cstdio>
+#include <omp.h>
+
+static constexpr int Threads = 8;
+static constexpr int Reps = 20;
+static constexpr int NTask = 4;
+
+int main() {
+ int errors = 0;
+
+ // 1. One encounter at a time. Untied tasks with a task scheduling point in
+ // them must still all run, on every replay.
+#pragma omp parallel num_threads(Threads)
+ {
+#pragma omp single
+ {
+ for (int rep = 0; rep < Reps; ++rep) {
+ int c = 0;
+#pragma omp taskgraph
+ {
+ for (int i = 0; i < NTask; ++i) {
+#pragma omp task untied shared(c)
+ {
+#pragma omp taskyield
+#pragma omp atomic
+ c++;
+ }
+ }
+ }
+ if (c != NTask) {
+ std::printf("single: rep %d ran %d of %d tasks\n", rep, c, NTask);
+ ++errors;
+ }
+ }
+ }
+ }
+
+ // 2. A dependent chain, so that each node's successor release has to happen
+ // exactly once and in order, on the last part of a multi-part task.
+#pragma omp parallel num_threads(Threads)
+ {
+#pragma omp single
+ {
+ for (int rep = 0; rep < Reps; ++rep) {
+ int seq = 0, out_of_order = 0;
+#pragma omp taskgraph
+ {
+ for (int i = 0; i < NTask; ++i) {
+#pragma omp task untied shared(seq, out_of_order) depend(inout : seq)
+ {
+#pragma omp taskyield
+ if (seq != i)
+ ++out_of_order;
+ seq++;
+ }
+ }
+ }
+ if (seq != NTask || out_of_order) {
+ std::printf("chain: rep %d reached %d of %d, %d out of order\n", rep,
+ seq, NTask, out_of_order);
+ ++errors;
+ }
+ }
+ }
+ }
+
+ // 3. No single: every thread encounters the directive and replays the shared
+ // record, each relocating the recorded tasks onto its own frame. A thread
+ // coming up short here means work landed on another thread's frame.
+ int short_threads = 0, grand = 0;
+#pragma omp parallel num_threads(Threads) reduction(+ : short_threads, grand)
+ {
+ int mine = 0;
+ for (int rep = 0; rep < Reps; ++rep) {
+ int c = 0;
+#pragma omp taskgraph
+ {
+ for (int i = 0; i < NTask; ++i) {
+#pragma omp task untied shared(c)
+ {
+#pragma omp taskyield
+#pragma omp atomic
+ c++;
+ }
+ }
+ }
+ mine += c;
+ }
+ grand += mine;
+ if (mine != Reps * NTask)
+ ++short_threads;
+ }
+ if (short_threads) {
+ std::printf("unsynchronised: %d of %d threads short, total %d of %d\n",
+ short_threads, Threads, grand, Reps * Threads * NTask);
+ ++errors;
+ }
+
+ std::printf("%s\n", errors ? "FAIL" : "PASS");
+ return errors != 0;
+}
+
+// CHECK-NOT: {{short|out of order|ran [0-9]+ of|reached [0-9]+ of}}
+// CHECK: PASS
>From 5044d0e16a19282c62563538e93f81871f07d4c9 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 24 Jul 2026 14:20:41 -0500
Subject: [PATCH 12/29] [Clang][OpenMP] Add taskgraph tests for 'default' on
replayable constructs
Adds tests covering default(none), default(shared) and
default(firstprivate) on replayable tasks and taskloops inside a
taskgraph, and those with a 'replayable' clause.
NOTE: taskgraph_default_firstprivate_taskloop.cpp's reduction
and taskgraph_default_firstprivate_reduction_codegen.cpp both
depend on the separate Sema fix "[Clang][OpenMP] Fix capture
of reduction variables under nested default(firstprivate)"
(https://github.com/llvm/llvm-project/pull/216329), which makes
isOpenMPCapturedByRef consult the capture level's default data-sharing
attribute rather than the innermost directive's. Without it the reduction
reads 0 and the CodeGen test fails, so that fix has to land first.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/216330
---
...default_firstprivate_reduction_codegen.cpp | 42 +++++
.../taskgraph_default_firstprivate.cpp | 124 +++++++++++++++
...askgraph_default_firstprivate_taskloop.cpp | 147 ++++++++++++++++++
.../taskgraph/taskgraph_default_none_1.cpp | 114 ++++++++++++++
.../taskgraph_default_none_taskloop.cpp | 105 +++++++++++++
.../taskgraph/taskgraph_default_shared.cpp | 88 +++++++++++
.../taskgraph_default_shared_taskloop.cpp | 85 ++++++++++
7 files changed, 705 insertions(+)
create mode 100644 clang/test/OpenMP/taskgraph_default_firstprivate_reduction_codegen.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_firstprivate.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_firstprivate_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_none_1.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_none_taskloop.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_shared.cpp
create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_shared_taskloop.cpp
diff --git a/clang/test/OpenMP/taskgraph_default_firstprivate_reduction_codegen.cpp b/clang/test/OpenMP/taskgraph_default_firstprivate_reduction_codegen.cpp
new file mode 100644
index 00000000000000..75c27202ae680f
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_default_firstprivate_reduction_codegen.cpp
@@ -0,0 +1,42 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+// expected-no-diagnostics
+
+// A taskloop with default(firstprivate) and a reduction, nested inside a
+// 'taskgraph' region. The reduction variable 'res' must be captured *by
+// reference* by the enclosing taskgraph region (its address stored into the
+// captured record) so the reduction result propagates back to it. A previous
+// bug consulted the innermost directive's default(firstprivate) when deciding
+// the enclosing region's capture kind and captured 'res' by copy, severing the
+// write-back (the caller then read the unmodified original, i.e. 0).
+
+#ifndef HEADER
+#define HEADER
+
+int run(int seed) {
+ int x = seed;
+ int res = 0;
+
+#pragma omp taskgraph graph_id(1)
+ {
+#pragma omp taskloop replayable num_tasks(4) default(firstprivate) reduction(+ : res)
+ for (int i = 0; i < 8; ++i)
+ res += x + i;
+ }
+
+ return res;
+}
+
+#endif
+
+// CHECK-LABEL: define {{.*}}@_Z3runi(
+// CHECK: %[[CAP:.*]] = alloca %struct.anon, align 8
+// The reduction variable is captured BY REFERENCE (a pointer is stored into the
+// taskgraph captured record's first field), not by copy.
+// CHECK: %[[FIELD:.*]] = getelementptr inbounds nuw %struct.anon, ptr %[[CAP]], i32 0, i32 0
+// CHECK-NEXT: store ptr %[[RES:.*]], ptr %[[FIELD]], align 8
+// CHECK: call void @__kmpc_taskgraph(
+// The caller reads the (reduced) value straight back out of that same storage.
+// CHECK: %[[RET:.*]] = load i32, ptr %[[RES]], align 4
+// CHECK-NEXT: ret i32 %[[RET]]
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate.cpp
new file mode 100644
index 00000000000000..b245b8a9a3a05a
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate.cpp
@@ -0,0 +1,124 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// default(firstprivate) on a replayable task. Every variable referenced in the
+// body without an explicit attribute becomes (plain) firstprivate; here that is
+// the automatic array 'local_arr'. 'SavedOff' is captured with the 'saved'
+// modifier and 'result' is explicitly shared so the outcome is observable.
+//
+// Record/replay semantics being verified:
+// - a plain firstprivate of a variable in the taskgraph's enclosing
+// environment takes the variable's value at the *replay* (OpenMP 6.0
+// section 14.3), so local_arr follows each call's seed;
+// - firstprivate(saved) likewise keeps the value snapshotted at recording,
+// even after the underlying static is mutated between encounters;
+// - the shared 'result' is relocated to the *current* call's frame on each
+// replay (so replays write to the live object, not the recorded one).
+//
+// The taskgraph is entered through call_with_depth() at a different recursion
+// depth for the recording and for each replay, and clobber_stack() overwrites
+// the abandoned frames in between. This is what makes the relocation clause
+// above load-bearing: if every encounter reused a single frame at one address,
+// a replay writing through the recorded (stale) shareds pointer would still
+// land on the live object by accident and the relocation path would not be
+// exercised at all.
+//
+// 'result' is an int in the frame rather than an int* supplied by the caller so
+// that a missing relocation fails as a stale write to a dead frame (leaving
+// 'result' at -1) instead of a wild store through a clobbered pointer.
+//
+// Both firstprivate list items are aggregates (an automatic array and a static
+// struct) so the aggregate save path is exercised rather than the
+// always-inlined intptr-sized scalar path.
+
+#include <cstdio>
+
+struct Off {
+ int a, b, c, d;
+};
+
+static Off SavedOff = {1, 2, 3, 4}; // a+b+c+d == 10
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+ volatile int scratch[4096];
+
+ for (int i = 0; i < 4096; ++i)
+ scratch[i] = base + i;
+
+ StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_default_firstprivate(int seed) {
+ int x = seed; // only used to anchor a dependence
+ int local_arr[3] = {seed, seed + 1, seed + 2};
+ int result = -1;
+
+#pragma omp taskgraph graph_id(9105)
+ {
+#pragma omp task default(firstprivate) shared(result) \
+ firstprivate(saved : SavedOff) depend(inout : x)
+ {
+ result = local_arr[0] + local_arr[1] + local_arr[2] + SavedOff.a +
+ SavedOff.b + SavedOff.c + SavedOff.d;
+ }
+ }
+
+ return result;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+ volatile int padding[128];
+
+ for (int i = 0; i < 128; ++i)
+ padding[i] = seed + depth + i;
+
+ StackSink += padding[(seed + depth) & 127];
+
+ if (depth == 0)
+ return run_default_firstprivate(seed);
+ return call_with_depth(seed, depth - 1);
+}
+
+int main() {
+ bool failed = false;
+
+ // Recording: local_arr == {1,2,3} (sum 6), SavedOff snapshot sum == 10.
+ const int a = call_with_depth(1, 0); // 6 + 10
+ if (a != 16) {
+ std::fprintf(stderr, "FAIL default(firstprivate) record a=%d expected=16\n",
+ a);
+ failed = true;
+ }
+
+ // Mutate the static; the saved snapshot must keep the recorded value, while
+ // the plain firstprivate array takes the current call's {seed, seed+1,
+ // seed+2}. The result must land in the frame of the current call, which
+ // sits at a different stack depth than the recording, proving the shared
+ // capture was relocated.
+ SavedOff = Off{9, 9, 9, 9};
+
+ clobber_stack(100 * 1000);
+ const int b = call_with_depth(100, 3); // 303 + 10
+ clobber_stack(5 * 1000);
+ const int c = call_with_depth(5, 6); // 18 + 10
+ if (b != 313 || c != 28) {
+ std::fprintf(
+ stderr, "FAIL default(firstprivate) replay b=%d c=%d expected=313/28\n",
+ b, c);
+ failed = true;
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr, "PASS default(firstprivate) task a=%d b=%d c=%d\n", a, b,
+ c);
+ return 0;
+}
+
+// CHECK: PASS default(firstprivate) task a=16 b=313 c=28
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate_taskloop.cpp
new file mode 100644
index 00000000000000..24bdd08cd9c1e5
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate_taskloop.cpp
@@ -0,0 +1,147 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// default(firstprivate) on a replayable taskloop. 'x' becomes (plain)
+// firstprivate of a variable in the taskgraph's enclosing environment and
+// therefore takes its value at each replay (OpenMP 6.0 section 14.3), in every
+// chunk; 'SavedBump' is captured via firstprivate(saved) (an aggregate, so the
+// aggregate save path is exercised); the loop iteration variable is
+// predetermined private; results are written both through an explicitly shared
+// output array (which must be relocated to the current call on replay) and a
+// reduction.
+//
+// The reduction here is a regression guard: default(firstprivate) previously
+// caused the enclosing taskgraph region to capture the reduction variable by
+// copy, so the reduced value never propagated back to the caller (it read as
+// 0).
+//
+// The shared output array is a local of run_default_firstprivate_taskloop
+// rather than the caller's array, and the taskgraph is entered through
+// call_with_depth() at a different recursion depth per encounter (with
+// clobber_stack() overwriting the abandoned frames in between). That is what
+// makes the relocation of the shared array load-bearing: reached at one fixed
+// depth the frame would recur at one address and a replay writing through the
+// recorded (stale) shareds pointer would hit the live array by accident. Using
+// a local rather than a caller-supplied pointer also means a missing relocation
+// shows up as a stale write to a dead frame (leaving the -1 initialiser behind)
+// instead of a wild store through a clobbered pointer.
+
+#include <cstdio>
+
+struct Bump {
+ int a, b;
+};
+
+static Bump SavedBump = {5, 7}; // a+b == 12
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+ volatile int scratch[4096];
+
+ for (int i = 0; i < 4096; ++i)
+ scratch[i] = base + i;
+
+ StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int
+run_default_firstprivate_taskloop(int seed, int *out_arr) {
+ int x = seed;
+ int total = 0;
+ int local_out[8];
+ for (int i = 0; i < 8; ++i)
+ local_out[i] = -1;
+
+#pragma omp taskgraph graph_id(9106)
+ {
+#pragma omp taskloop replayable num_tasks(4) default(firstprivate) \
+ shared(local_out) firstprivate(saved : SavedBump) reduction(+ : total)
+ for (int i = 0; i < 8; ++i) {
+ int v = x + i + SavedBump.a + SavedBump.b;
+ local_out[i] = v;
+ total += v;
+ }
+ }
+
+ for (int i = 0; i < 8; ++i)
+ out_arr[i] = local_out[i];
+
+ return total;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int *out_arr,
+ int depth) {
+ volatile int padding[128];
+
+ for (int i = 0; i < 128; ++i)
+ padding[i] = seed + depth + i;
+
+ StackSink += padding[(seed + depth) & 127];
+
+ if (depth == 0)
+ return run_default_firstprivate_taskloop(seed, out_arr);
+ return call_with_depth(seed, out_arr, depth - 1);
+}
+
+static bool check(const int *arr, int base, const char *tag) {
+ for (int i = 0; i < 8; ++i) {
+ if (arr[i] != base + i) {
+ std::fprintf(stderr,
+ "FAIL default(firstprivate) taskloop %s arr[%d]=%d "
+ "expected=%d\n",
+ tag, i, arr[i], base + i);
+ return false;
+ }
+ }
+ return true;
+}
+
+int main() {
+ bool ok = true;
+ int o1[8], o2[8];
+ for (int i = 0; i < 8; ++i) {
+ o1[i] = -1;
+ o2[i] = -1;
+ }
+
+ // Recording: out_arr[i] = x(10) + i + 12 = 22 + i; total = sum(22+i) = 204.
+ const int t1 = call_with_depth(10, o1, 0);
+ ok &= check(o1, 22, "record");
+ if (t1 != 204) {
+ std::fprintf(stderr,
+ "FAIL default(firstprivate) taskloop record total=%d "
+ "expected=204\n",
+ t1);
+ ok = false;
+ }
+
+ // Mutate the static; the saved 'SavedBump' (frozen at sum 12) keeps its
+ // recorded value while the plain firstprivate 'x' takes the current call's
+ // 99, so the replay produces 111 + i (into the current call's array, reached
+ // at a different stack depth) and the reduction total is sum(111+i) = 916.
+ SavedBump = Bump{100, 100};
+
+ clobber_stack(99 * 1000);
+ const int t2 = call_with_depth(99, o2, 4);
+ ok &= check(o2, 111, "replay");
+ if (t2 != 916) {
+ std::fprintf(stderr,
+ "FAIL default(firstprivate) taskloop replay total=%d "
+ "expected=916\n",
+ t2);
+ ok = false;
+ }
+
+ if (!ok)
+ return 1;
+
+ std::fprintf(stderr, "PASS default(firstprivate) taskloop total=%d/%d\n", t1,
+ t2);
+ return 0;
+}
+
+// CHECK: PASS default(firstprivate) taskloop total=204/916
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_none_1.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_none_1.cpp
new file mode 100644
index 00000000000000..24448d92fdf7db
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_none_1.cpp
@@ -0,0 +1,114 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A replayable task with default(none) forces every referenced variable to be
+// given an explicit data-sharing attribute. This used to crash Clang's Sema
+// (llvm_unreachable "Unexpected clause") because the implicit-DSA check for
+// default(none) walked the clause list and had no case for the taskgraph
+// 'replayable' clause. Beyond just compiling, this exercises the two runtime
+// mechanisms that must keep working under default(none):
+// - shared-by-reference relocation: the shared 'x'/'out' locals live on a
+// stack frame that moves between the recording call and each replay call,
+// and the shared 'x' must be re-read from the *current* call;
+// - firstprivate(saved) snapshotting of a variable that is mutated between
+// encounters. A static ARRAY (larger than a pointer) is used on purpose so
+// the aggregate save path -- rather than the always-inlined intptr-sized
+// scalar path -- is exercised.
+//
+// The frame only moves because each encounter enters the taskgraph through
+// call_with_depth() at a different recursion depth, with clobber_stack()
+// overwriting the abandoned frames in between. Called straight from main() the
+// frame would recur at one address every time and a replay dereferencing the
+// recorded (stale) shareds pointer would still find the live 'x'/'out', so the
+// relocation half of the above would go untested.
+
+#include <cstdio>
+
+static int SavedTable[5] = {2, 4, 6, 8, 10}; // sum == 30
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+ volatile int scratch[4096];
+
+ for (int i = 0; i < 4096; ++i)
+ scratch[i] = base + i;
+
+ StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int sum_table(const int *t) {
+ int s = 0;
+ for (int i = 0; i < 5; ++i)
+ s += t[i];
+ return s;
+}
+
+__attribute__((noinline)) static int run_default_none(int seed) {
+ int x = seed;
+ int out = -1;
+
+#pragma omp taskgraph graph_id(9101)
+ {
+#pragma omp task default(none) shared(x, out) firstprivate(saved : SavedTable) \
+ depend(inout : x)
+ {
+ out = x + sum_table(SavedTable);
+ }
+ }
+
+ return out;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+ volatile int padding[128];
+
+ for (int i = 0; i < 128; ++i)
+ padding[i] = seed + depth + i;
+
+ StackSink += padding[(seed + depth) & 127];
+
+ if (depth == 0)
+ return run_default_none(seed);
+ return call_with_depth(seed, depth - 1);
+}
+
+int main() {
+ bool failed = false;
+
+ // Recording: snapshots SavedTable (sum 30) into the task's saved slot.
+ const int first = call_with_depth(1, 0); // 1 + 30
+ if (first != 31) {
+ std::fprintf(stderr, "FAIL default(none) record out=%d expected=31\n",
+ first);
+ failed = true;
+ }
+
+ // Mutate the whole array; the saved snapshot (sum 30) must keep winning while
+ // the shared 'x' is re-read from the current call.
+ for (int i = 0; i < 5; ++i)
+ SavedTable[i] = 100;
+
+ clobber_stack(50 * 1000);
+ const int second = call_with_depth(50, 3); // 50 + 30
+ clobber_stack(7 * 1000);
+ const int third = call_with_depth(7, 6); // 7 + 30
+ if (second != 80 || third != 37) {
+ std::fprintf(
+ stderr, "FAIL default(none) replay second=%d third=%d expected=80/37\n",
+ second, third);
+ failed = true;
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr, "PASS default(none) task first=%d second=%d third=%d\n",
+ first, second, third);
+ return 0;
+}
+
+// CHECK: PASS default(none) task first=31 second=80 third=37
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_none_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_none_taskloop.cpp
new file mode 100644
index 00000000000000..0b072c0773ff77
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_none_taskloop.cpp
@@ -0,0 +1,105 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// default(none) on a replayable taskloop. As with the task variant this used
+// to crash Sema. The loop iteration variable is predetermined private and so
+// needs no explicit clause; everything else is spelled out. The shared 'x'
+// local is relocated (and re-read) between the recording call and the replays,
+// 'res' is a reduction, and 'SavedBias' is snapshotted with firstprivate(saved)
+// and must survive being mutated between encounters. An aggregate (struct) is
+// used for the saved list item so the aggregate save path is exercised rather
+// than the always-inlined intptr-sized scalar path.
+//
+// Each encounter enters the taskgraph through call_with_depth() at a different
+// recursion depth, with clobber_stack() overwriting the abandoned frames in
+// between, so the frame holding 'x' and 'res' really does move. Entering from
+// main() directly would reuse one frame at one address, and a replay reading
+// 'x' through the recorded (stale) shareds pointer would still find the live
+// value by accident.
+
+#include <cstdio>
+
+struct Bias3 {
+ int p, q, r;
+};
+
+static Bias3 SavedBias = {1, 2, 3}; // p+q+r == 6
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+ volatile int scratch[4096];
+
+ for (int i = 0; i < 4096; ++i)
+ scratch[i] = base + i;
+
+ StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_default_none_taskloop(int seed) {
+ int x = seed;
+ int res = 0;
+
+#pragma omp taskgraph graph_id(9102)
+ {
+#pragma omp taskloop replayable num_tasks(8) default(none) shared(x) \
+ firstprivate(saved : SavedBias) reduction(+ : res)
+ for (int i = 0; i < 16; ++i)
+ res += x + i + SavedBias.p + SavedBias.q + SavedBias.r;
+ }
+
+ return res;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+ volatile int padding[128];
+
+ for (int i = 0; i < 128; ++i)
+ padding[i] = seed + depth + i;
+
+ StackSink += padding[(seed + depth) & 127];
+
+ if (depth == 0)
+ return run_default_none_taskloop(seed);
+ return call_with_depth(seed, depth - 1);
+}
+
+int main() {
+ bool failed = false;
+
+ // sum_{i=0}^{15} (x + i + 6) = 16*x + 120 + 96 = 16*x + 216.
+ const int first = call_with_depth(1, 0); // 16 + 216
+ if (first != 232) {
+ std::fprintf(stderr,
+ "FAIL default(none) taskloop record res=%d expected=232\n",
+ first);
+ failed = true;
+ }
+
+ SavedBias = Bias3{100, 100, 100};
+
+ clobber_stack(100 * 1000);
+ const int second = call_with_depth(100, 3); // 1600 + 216
+ clobber_stack(5 * 1000);
+ const int third = call_with_depth(5, 6); // 80 + 216
+ if (second != 1816 || third != 296) {
+ std::fprintf(stderr,
+ "FAIL default(none) taskloop replay second=%d third=%d "
+ "expected=1816/296\n",
+ second, third);
+ failed = true;
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr,
+ "PASS default(none) taskloop first=%d second=%d third=%d\n",
+ first, second, third);
+ return 0;
+}
+
+// CHECK: PASS default(none) taskloop first=232 second=1816 third=296
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_shared.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_shared.cpp
new file mode 100644
index 00000000000000..3a2a0a1ed5189e
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_shared.cpp
@@ -0,0 +1,88 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// default(shared) on a replayable task: 'x' and 'out' get an implicit shared
+// attribute rather than being listed explicitly. Because they are shared by
+// reference and live on a stack frame that moves between the recording call and
+// the replay calls, this specifically exercises shared-by-reference relocation
+// under an implicit (default) data-sharing attribute.
+//
+// The frame only actually moves because the taskgraph is entered through
+// call_with_depth() at a different recursion depth per encounter, with
+// clobber_stack() overwriting the abandoned frames in between. Calling
+// run_default_shared() directly from main() would reuse one frame at one
+// address on every encounter, so a replay reading and writing through the
+// recorded (stale) shareds pointer would still hit the live objects by accident
+// and relocation would not be exercised at all.
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+ volatile int scratch[4096];
+
+ for (int i = 0; i < 4096; ++i)
+ scratch[i] = base + i;
+
+ StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_default_shared(int seed) {
+ int x = seed;
+ int out = -1;
+
+#pragma omp taskgraph graph_id(9103)
+ {
+#pragma omp task default(shared) depend(inout : x)
+ {
+ x += 5;
+ out = x;
+ }
+ }
+
+ return out;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+ volatile int padding[128];
+
+ for (int i = 0; i < 128; ++i)
+ padding[i] = seed + depth + i;
+
+ StackSink += padding[(seed + depth) & 127];
+
+ if (depth == 0)
+ return run_default_shared(seed);
+ return call_with_depth(seed, depth - 1);
+}
+
+int main() {
+ bool failed = false;
+
+ const int first = call_with_depth(1, 0);
+ clobber_stack(100 * 1000);
+ const int second = call_with_depth(100, 3);
+ clobber_stack(5 * 1000);
+ const int third = call_with_depth(5, 6);
+ if (first != 6 || second != 105 || third != 10) {
+ std::fprintf(stderr,
+ "FAIL default(shared) task first=%d second=%d third=%d "
+ "expected=6/105/10\n",
+ first, second, third);
+ failed = true;
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr,
+ "PASS default(shared) task first=%d second=%d third=%d\n", first,
+ second, third);
+ return 0;
+}
+
+// CHECK: PASS default(shared) task first=6 second=105 third=10
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_shared_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_shared_taskloop.cpp
new file mode 100644
index 00000000000000..ec917f2348aad8
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_shared_taskloop.cpp
@@ -0,0 +1,85 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// default(shared) on a replayable taskloop: 'x' is implicitly shared and
+// relocated across replays; 'res' is an explicit reduction (which overrides the
+// default) and the loop variable is predetermined private.
+//
+// Both 'x' and 'res' live in run_default_shared_taskloop's frame, and the
+// taskgraph is entered through call_with_depth() at a different recursion depth
+// per encounter (with clobber_stack() overwriting the abandoned frames in
+// between) so that frame genuinely moves. Entering from main() directly would
+// reuse one frame at one address, letting a replay that read 'x' through the
+// recorded (stale) shareds pointer land on the live object by accident, so the
+// relocation claim above would not be tested.
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+ volatile int scratch[4096];
+
+ for (int i = 0; i < 4096; ++i)
+ scratch[i] = base + i;
+
+ StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_default_shared_taskloop(int seed) {
+ int x = seed;
+ int res = 0;
+
+#pragma omp taskgraph graph_id(9104)
+ {
+#pragma omp taskloop replayable num_tasks(8) default(shared) reduction(+ : res)
+ for (int i = 0; i < 16; ++i)
+ res += x + i;
+ }
+
+ return res;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+ volatile int padding[128];
+
+ for (int i = 0; i < 128; ++i)
+ padding[i] = seed + depth + i;
+
+ StackSink += padding[(seed + depth) & 127];
+
+ if (depth == 0)
+ return run_default_shared_taskloop(seed);
+ return call_with_depth(seed, depth - 1);
+}
+
+int main() {
+ bool failed = false;
+
+ // sum_{i=0}^{15} (x + i) = 16*x + 120.
+ const int first = call_with_depth(1, 0); // 16 + 120
+ clobber_stack(100 * 1000);
+ const int second = call_with_depth(100, 3); // 1600 + 120
+ clobber_stack(5 * 1000);
+ const int third = call_with_depth(5, 6); // 80 + 120
+ if (first != 136 || second != 1720 || third != 200) {
+ std::fprintf(stderr,
+ "FAIL default(shared) taskloop first=%d second=%d third=%d "
+ "expected=136/1720/200\n",
+ first, second, third);
+ failed = true;
+ }
+
+ if (failed)
+ return 1;
+
+ std::fprintf(stderr,
+ "PASS default(shared) taskloop first=%d second=%d third=%d\n",
+ first, second, third);
+ return 0;
+}
+
+// CHECK: PASS default(shared) taskloop first=136 second=1720 third=200
>From 357cc21a9933ef2ad53b983843747f40bacfea10 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 19 Aug 2026 10:39:06 -0500
Subject: [PATCH 13/29] [OpenMP][Offload] Taskgraph target directive recording:
runtime parts
This patch contains the runtime parts of recording target constructs into
a taskgraph, complementing the earlier patches supporting recording for
host tasks. This is the mechanism that captures target kernel launches,
'target enter data', 'target exit data' and 'target update' directives
into the recorded taskgraph. As for plain tasks, separate entry points
for taskgraph recording are used vs. those used for immediate execution.
When recording a taskgraph, responsibility for capturing which target
constructs are used is handled by libomp. Because the target constructs
must still execute, this means we must still forward the operations to
libomptarget as we see them. This is a difference from non-taskgraph
execution, where the compiled program normally calls direct into
libomptarget, and libomp isn't involved. For simplicity, during recording
no "hidden helper" tasks are used. Instead, target constructs execute
synchronously (i.e. ignoring "nowait"), which is sufficient to get through
the recording process and capture the dependencies between offload tasks.
Recording a target construct deep-copies its kernel arguments or
map arrays, so that the record outlives the encountering task's
stack. The copies are made by two new libomptarget entry points,
__tgt_taskgraph_dup_kernel_args and __tgt_taskgraph_dup_data_args, each
called twice (once to size a single block, once to fill it), so that
libomp does not need to know the layout of the kernel-arguments blob.
A non-contiguous 'target update' passes a pointer to an array of stride
descriptors in args[i], which clang emits in the encountering function's
frame, so __tgt_taskgraph_dup_data_args copies the descriptors into the
same block too.
Similar to previous patches which perform relocation to replay host
tasks at different stack depths (or from different stacks), new
compiler-generated relocation helpers need to be emitted for target
constructs also: separate helpers are needed for kernels and for
data-mapping operations, since they work on differently-shaped data.
The mechanism is outlined here but the actual helpers aren't introduced
until later in the series.
A small change has been made to the kernel-launch path. The current
code overwrites KernelArgs.NumArgs during kernel launch. Because for
taskgraphs kernel args are reused from one replay to the next, it's
helpful to undo that change surgically (via the "RAII" SaveAndRestore
method) so we get the same starting value on the subsequent launch.
That avoids needing to duplicate and reinitialise the whole kernel
args array.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/225713
---
offload/include/omptarget.h | 14 +
offload/libomptarget/exports | 2 +
offload/libomptarget/interface.cpp | 175 +++++++++++
offload/libomptarget/omptarget.cpp | 11 +-
openmp/runtime/src/kmp.h | 102 ++++++-
openmp/runtime/src/kmp_taskdeps.cpp | 51 +++-
openmp/runtime/src/kmp_tasking.cpp | 453 +++++++++++++++++++++++++++-
7 files changed, 782 insertions(+), 26 deletions(-)
diff --git a/offload/include/omptarget.h b/offload/include/omptarget.h
index 01be2d9cae6b50..573127406a0ed8 100644
--- a/offload/include/omptarget.h
+++ b/offload/include/omptarget.h
@@ -462,6 +462,20 @@ void *__tgt_get_mapped_ptr(int64_t DeviceId, const void *HostPtr);
// code for details.
void __tgt_register_rpc_callback(unsigned (*Callback)(void *, unsigned));
+/// Duplicate kernel args at capture time. Uses a two-pass measure/allocate
+/// strategy. Used so libomp doesn't have to understand kernel arg layout.
+void __tgt_taskgraph_dup_kernel_args(void *Dst, void *Src, size_t *AllocSize);
+
+/// Duplicate the map arrays of a target data construct at capture time.
+///
+/// This is here mainly for symmetry with __tgt_taskgraph_dup_kernel_args, and
+/// is called in the same two-pass manner. In this case both sides already know
+/// the data layout though.
+void __tgt_taskgraph_dup_data_args(void *Dst, int32_t ArgNum, void ***ArgsBase,
+ void ***Args, int64_t **ArgSizes,
+ int64_t **ArgTypes, void ***ArgNames,
+ void ***ArgMappers, size_t *AllocSize);
+
#ifdef __cplusplus
}
#endif
diff --git a/offload/libomptarget/exports b/offload/libomptarget/exports
index 91d29fe41679cd..037207d67da554 100644
--- a/offload/libomptarget/exports
+++ b/offload/libomptarget/exports
@@ -31,6 +31,8 @@ VERS1.0 {
__tgt_target_kernel_nowait;
__tgt_target_nowait_query;
__tgt_target_kernel_replay;
+ __tgt_taskgraph_dup_kernel_args;
+ __tgt_taskgraph_dup_data_args;
__tgt_activate_record_replay;
__tgt_mapper_num_components;
__tgt_push_mapper_component;
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index 028d94e80fe393..1170618a0c2e08 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -25,11 +25,13 @@
#include "Utils/ExponentialBackoff.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
+#include "llvm/Support/MathExtras.h"
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
+#include <cstring>
#include <memory>
#include <vector>
@@ -672,3 +674,176 @@ EXTERN void *__tgt_get_mapped_ptr(int64_t DeviceId, const void *HostPtr) {
return const_cast<void *>(HostPtr);
return TargetPtr;
}
+
+EXTERN void __tgt_taskgraph_dup_kernel_args(void *Dst, void *Src,
+ size_t *AllocSize) {
+ // Deep-copy a kernel-arguments blob into a single caller-provided block, so
+ // that libomp can capture a target launch without knowing the layout of
+ // KernelArgsTy or of the six argument arrays hanging off it. Called twice:
+ // once with a null Dst to compute the size of the block (returned in
+ // AllocSize), then again to fill a block of that size.
+ bool SizeOnly = (Dst == nullptr);
+ KernelArgsTy *KernArgs = static_cast<KernelArgsTy *>(Src);
+ size_t N = KernArgs->NumArgs;
+ // ArgNames and ArgMappers are optional; a launch with neither debug names nor
+ // user-defined mappers leaves them null. The copy has to preserve that, or
+ // every argument would appear to carry a mapper (pointing at uninitialized
+ // bytes of the block). Both passes read the same blob, so both agree on
+ // whether the arrays are there.
+ const bool HasNames = KernArgs->ArgNames != nullptr;
+ const bool HasMappers = KernArgs->ArgMappers != nullptr;
+ if (SizeOnly) {
+ size_t Total = 0;
+ auto CountAligned = [&](size_t Size, size_t Align) {
+ Total = llvm::alignTo(Total, Align) + Size;
+ };
+ CountAligned(sizeof(KernelArgsTy), alignof(KernelArgsTy));
+ // ArgBasePtrs
+ CountAligned(N * sizeof(void *), alignof(void *));
+ // ArgPtrs
+ CountAligned(N * sizeof(void *), alignof(void *));
+ // ArgSizes
+ CountAligned(N * sizeof(uint64_t), alignof(uint64_t));
+ // ArgTypes
+ CountAligned(N * sizeof(uint64_t), alignof(uint64_t));
+ // ArgNames
+ if (HasNames)
+ CountAligned(N * sizeof(void *), alignof(void *));
+ // ArgMappers
+ if (HasMappers)
+ CountAligned(N * sizeof(void *), alignof(void *));
+ *AllocSize = Total;
+ } else {
+ size_t Allocated = 0;
+ auto BumpAllocDup = [&](size_t Size, size_t Align,
+ void *Src = nullptr) -> void * {
+ size_t ThisBlockIdx = llvm::alignTo(Allocated, Align);
+ char *Block = &(static_cast<char *>(Dst))[ThisBlockIdx];
+ Allocated = ThisBlockIdx + Size;
+ if (Src)
+ std::memcpy(Block, Src, Size);
+ return Block;
+ };
+ auto *DupArgs = static_cast<KernelArgsTy *>(
+ BumpAllocDup(sizeof(KernelArgsTy), alignof(KernelArgsTy), Src));
+ DupArgs->ArgBasePtrs = static_cast<void **>(BumpAllocDup(
+ N * sizeof(void *), alignof(void *), KernArgs->ArgBasePtrs));
+ DupArgs->ArgPtrs = static_cast<void **>(
+ BumpAllocDup(N * sizeof(void *), alignof(void *), KernArgs->ArgPtrs));
+ DupArgs->ArgSizes = static_cast<int64_t *>(BumpAllocDup(
+ N * sizeof(int64_t), alignof(uint64_t), KernArgs->ArgSizes));
+ DupArgs->ArgTypes = static_cast<int64_t *>(BumpAllocDup(
+ N * sizeof(int64_t), alignof(uint64_t), KernArgs->ArgTypes));
+ DupArgs->ArgNames =
+ HasNames ? static_cast<void **>(BumpAllocDup(
+ N * sizeof(void *), alignof(void *), KernArgs->ArgNames))
+ : nullptr;
+ DupArgs->ArgMappers =
+ HasMappers
+ ? static_cast<void **>(BumpAllocDup(
+ N * sizeof(void *), alignof(void *), KernArgs->ArgMappers))
+ : nullptr;
+ assert(Allocated == *AllocSize);
+ }
+}
+
+EXTERN void __tgt_taskgraph_dup_data_args(void *Dst, int32_t ArgNum,
+ void ***ArgsBase, void ***Args,
+ int64_t **ArgSizes,
+ int64_t **ArgTypes, void ***ArgNames,
+ void ***ArgMappers,
+ size_t *AllocSize) {
+ // Deep-copy the map arrays of a target data construct into a single
+ // caller-provided block, so that libomp can capture one without knowing what
+ // an element of them means. Called twice, as
+ // __tgt_taskgraph_dup_kernel_args is: once with a null Dst to compute the
+ // size of the block (returned in AllocSize), then again to fill a block of
+ // that size, which is when the in-out array parameters are repointed into it.
+ bool SizeOnly = (Dst == nullptr);
+ size_t N = ArgNum;
+ // Copying the arrays is not enough on its own for a non-contiguous update.
+ // Its args[i] is a pointer to a stride descriptor rather than to mapped
+ // data, and clang emits that descriptor into the encountering function's
+ // frame, so a copy of the pointer dangles as soon as the recording encounter
+ // returns. The descriptors are copied into the same block and the copied
+ // args[i] repointed at them, which also puts them beyond the reach of
+ // relocation for good: they are runtime-owned from here on and, being pure
+ // offsets, counts and strides, have nothing frame-dependent left in them to
+ // rewrite. arg_sizes[i] is the dimension count for such an entry, not a
+ // byte size.
+ auto NonContigDims = [&](size_t Idx) -> size_t {
+ return ((*ArgTypes)[Idx] & OMP_TGT_MAPTYPE_NON_CONTIG)
+ ? (size_t)(*ArgSizes)[Idx]
+ : 0;
+ };
+ // ArgNames and ArgMappers are optional; a construct with neither debug names
+ // nor user-defined mappers leaves them null, and the copy has to preserve
+ // that rather than hand back pointers to uninitialized bytes of the block.
+ // Both calls are given the same arrays, so both agree on whether they are
+ // there.
+ const bool HasNames = *ArgNames != nullptr;
+ const bool HasMappers = *ArgMappers != nullptr;
+ if (SizeOnly) {
+ size_t Total = 0;
+ auto CountAligned = [&](size_t Size, size_t Align) {
+ Total = llvm::alignTo(Total, Align) + Size;
+ };
+ // ArgsBase
+ CountAligned(N * sizeof(void *), alignof(void *));
+ // Args
+ CountAligned(N * sizeof(void *), alignof(void *));
+ // ArgSizes
+ CountAligned(N * sizeof(int64_t), alignof(int64_t));
+ // ArgTypes
+ CountAligned(N * sizeof(int64_t), alignof(int64_t));
+ if (HasNames)
+ CountAligned(N * sizeof(void *), alignof(void *));
+ if (HasMappers)
+ CountAligned(N * sizeof(void *), alignof(void *));
+ // One descriptor array per non-contiguous entry.
+ for (size_t I = 0; I < N; I++)
+ if (size_t Dims = NonContigDims(I))
+ CountAligned(Dims * sizeof(__tgt_target_non_contig),
+ alignof(__tgt_target_non_contig));
+ *AllocSize = Total;
+ return;
+ }
+
+ size_t Allocated = 0;
+ auto BumpAllocDup = [&](size_t Size, size_t Align, void *Src) -> void * {
+ size_t ThisBlockIdx = llvm::alignTo(Allocated, Align);
+ char *Block = &(static_cast<char *>(Dst))[ThisBlockIdx];
+ Allocated = ThisBlockIdx + Size;
+ if (Src)
+ std::memcpy(Block, Src, Size);
+ return Block;
+ };
+ auto *DupArgsBase = static_cast<void **>(
+ BumpAllocDup(N * sizeof(void *), alignof(void *), *ArgsBase));
+ auto *DupArgs = static_cast<void **>(
+ BumpAllocDup(N * sizeof(void *), alignof(void *), *Args));
+ auto *DupArgSizes = static_cast<int64_t *>(
+ BumpAllocDup(N * sizeof(int64_t), alignof(int64_t), *ArgSizes));
+ auto *DupArgTypes = static_cast<int64_t *>(
+ BumpAllocDup(N * sizeof(int64_t), alignof(int64_t), *ArgTypes));
+ auto *DupArgNames =
+ HasNames ? static_cast<void **>(BumpAllocDup(N * sizeof(void *),
+ alignof(void *), *ArgNames))
+ : nullptr;
+ auto *DupArgMappers =
+ HasMappers ? static_cast<void **>(BumpAllocDup(
+ N * sizeof(void *), alignof(void *), *ArgMappers))
+ : nullptr;
+ for (size_t I = 0; I < N; I++)
+ if (size_t Dims = NonContigDims(I))
+ DupArgs[I] = BumpAllocDup(Dims * sizeof(__tgt_target_non_contig),
+ alignof(__tgt_target_non_contig), (*Args)[I]);
+ assert(Allocated == *AllocSize);
+
+ *ArgsBase = DupArgsBase;
+ *Args = DupArgs;
+ *ArgSizes = DupArgSizes;
+ *ArgTypes = DupArgTypes;
+ *ArgNames = DupArgNames;
+ *ArgMappers = DupArgMappers;
+}
diff --git a/offload/libomptarget/omptarget.cpp b/offload/libomptarget/omptarget.cpp
index b03ec4afa0891e..0038796447f84b 100644
--- a/offload/libomptarget/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -32,6 +32,7 @@
#include "llvm/ADT/bit.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/SaveAndRestore.h"
#include <algorithm>
#include <cassert>
@@ -2282,11 +2283,11 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
PrivateArgumentManagerTy PrivateArgumentManager(Device, AsyncInfo);
- int NumClangLaunchArgs = KernelArgs.NumArgs;
+ llvm::SaveAndRestore<uint32_t> NumClangLaunchArgs(KernelArgs.NumArgs);
int Ret = OFFLOAD_SUCCESS;
- if (NumClangLaunchArgs) {
+ if (NumClangLaunchArgs.get()) {
// Process data, such as data mapping, before launching the kernel
- Ret = processDataBefore(Loc, DeviceId, HostPtr, NumClangLaunchArgs,
+ Ret = processDataBefore(Loc, DeviceId, HostPtr, NumClangLaunchArgs.get(),
KernelArgs.ArgBasePtrs, KernelArgs.ArgPtrs,
KernelArgs.ArgSizes, KernelArgs.ArgTypes,
KernelArgs.ArgNames, KernelArgs.ArgMappers, TgtArgs,
@@ -2336,10 +2337,10 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
return OFFLOAD_FAIL;
}
- if (NumClangLaunchArgs) {
+ if (NumClangLaunchArgs.get()) {
// Transfer data back and deallocate target memory for (first-)private
// variables
- Ret = processDataAfter(Loc, DeviceId, HostPtr, NumClangLaunchArgs,
+ Ret = processDataAfter(Loc, DeviceId, HostPtr, NumClangLaunchArgs.get(),
KernelArgs.ArgBasePtrs, KernelArgs.ArgPtrs,
KernelArgs.ArgSizes, KernelArgs.ArgTypes,
KernelArgs.ArgNames, KernelArgs.ArgMappers,
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index a82925fc7a13e1..c6cc4c999fcc72 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2703,21 +2703,94 @@ enum kmp_taskgraph_region_type {
TASKGRAPH_REGION_EXIT,
TASKGRAPH_REGION_NODE,
TASKGRAPH_REGION_WAIT,
+ // Target constructs. These carry a task node like NODE/WAIT. Keep the
+ // TARGET codes contiguous e.g. for __kmp_taskgraph_region_is_target below.
+ TASKGRAPH_REGION_TARGET,
+ TASKGRAPH_REGION_TARGET_ENTER_DATA,
+ TASKGRAPH_REGION_TARGET_EXIT_DATA,
+ TASKGRAPH_REGION_TARGET_UPDATE,
TASKGRAPH_REGION_PARALLEL,
TASKGRAPH_REGION_EXCLUSIVE,
TASKGRAPH_REGION_SEQUENTIAL,
TASKGRAPH_REGION_IRREDUCIBLE
};
+// True for the four target / target-data region types.
+static inline bool
+__kmp_taskgraph_region_is_target(enum kmp_taskgraph_region_type type) {
+ return type >= TASKGRAPH_REGION_TARGET &&
+ type <= TASKGRAPH_REGION_TARGET_UPDATE;
+}
+
+// True for regions that use the task arm of kmp_taskgraph_region (i.e. carry
+// a node): plain task nodes, taskwait nodes, and all target nodes.
+static inline bool
+__kmp_taskgraph_region_is_task_arm(enum kmp_taskgraph_region_type type) {
+ return type == TASKGRAPH_REGION_NODE || type == TASKGRAPH_REGION_WAIT ||
+ __kmp_taskgraph_region_is_target(type);
+}
+
+// Case labels for the four target region types, for use in switch statements.
+#define KMP_TASKGRAPH_REGION_TARGET_CASES \
+ case TASKGRAPH_REGION_TARGET: \
+ case TASKGRAPH_REGION_TARGET_ENTER_DATA: \
+ case TASKGRAPH_REGION_TARGET_EXIT_DATA: \
+ case TASKGRAPH_REGION_TARGET_UPDATE
+
+// Target relocation callbacks: rewrite moved host pointers and refresh
+// by-value firstprivates in the captured target arguments before a recorded
+// target region is re-issued on replay. Similar in principle to
+// kmp_task_relocate_t, but operating on differently-shaped data: a kernel
+// construct's callback is handed its kernel-arguments blob, and a data
+// construct's its two arrays of addresses.
+typedef void (*kmp_target_kernel_relocate_t)(void *kernel_args,
+ void *outer_captures);
+typedef void (*kmp_target_data_relocate_t)(void **args_base, void **args,
+ void *outer_captures);
+
+// Separately-allocated metadata for a target/target-data regions.
+typedef struct kmp_taskgraph_target_node {
+ enum kmp_taskgraph_region_type kind;
+ kmp_int64 device_id;
+ union {
+ // TASKGRAPH_REGION_TARGET
+ struct {
+ kmp_target_kernel_relocate_t relocate;
+ kmp_int32 num_teams;
+ kmp_int32 thread_limit;
+ void *host_ptr;
+ void *kernel_args; // deep copy of the kernel-arguments struct
+ } kernel;
+ // TASKGRAPH_REGION_TARGET_{ENTER,EXIT}_DATA / _UPDATE. The arrays are deep
+ // copies sized by arg_num, made by libomptarget into a block referenced by
+ // the opaque 'arena' pointer.
+ struct {
+ kmp_target_data_relocate_t relocate;
+ kmp_int32 arg_num;
+ void *arena;
+ void **args_base;
+ void **args;
+ kmp_int64 *arg_sizes;
+ kmp_int64 *arg_types;
+ void **arg_names;
+ void **arg_mappers;
+ } data;
+ } u;
+} kmp_taskgraph_target_node_t;
+
typedef struct kmp_taskgraph_node {
kmp_task_t *task;
bool taskloop_task;
- // True for undeferred (included) tasks, which block execution of the
- // encountering task until they complete -- e.g. a taskwait without 'nowait'.
- // Everything recorded after one is ordered after it.
+ // True for undeferred (included) tasks. For example target constructs within
+ // a taskgraph are 'included', which means that they block execution of the
+ // encountering task. This allows e.g. "target enter data", "target", "target
+ // exit data" constructs with no explicit data dependencies between them to
+ // execute sequentially in taskgraph context as a user would expect.
bool undeferred;
kmp_task_relocate_t relocate;
kmp_taskgraph_reduce_input_data_t *reduce_input;
+ // Non-NULL iff this node represents a target / target-data construct.
+ kmp_taskgraph_target_node_t *target;
kmp_depend_info_t *dep_list;
kmp_int32 ndeps;
// This is a control dependency. If not -1, it is the index of the
@@ -4609,6 +4682,29 @@ KMP_EXPORT void __kmpc_taskgraph_taskwait(ident_t *loc_ref, kmp_int32 gtid,
kmp_int32 has_no_wait);
KMP_EXPORT void *__kmpc_taskgraph_taskred_init(kmp_int32 gtid, kmp_int32 num,
void *data);
+KMP_EXPORT kmp_int32 __kmpc_taskgraph_target(
+ ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_int32 has_no_wait, kmp_int64 device_id,
+ kmp_int32 num_teams, kmp_int32 thread_limit, void *host_ptr,
+ void *kernel_args, kmp_target_kernel_relocate_t reloc);
+KMP_EXPORT void __kmpc_taskgraph_target_enter_data(
+ ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_int32 has_no_wait, kmp_int64 device_id,
+ kmp_int32 arg_num, void **args_base, void **args, kmp_int64 *arg_sizes,
+ kmp_int64 *arg_types, void **arg_names, void **arg_mappers,
+ kmp_target_data_relocate_t reloc);
+KMP_EXPORT void __kmpc_taskgraph_target_exit_data(
+ ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_int32 has_no_wait, kmp_int64 device_id,
+ kmp_int32 arg_num, void **args_base, void **args, kmp_int64 *arg_sizes,
+ kmp_int64 *arg_types, void **arg_names, void **arg_mappers,
+ kmp_target_data_relocate_t reloc);
+KMP_EXPORT void __kmpc_taskgraph_target_update(
+ ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_int32 has_no_wait, kmp_int64 device_id,
+ kmp_int32 arg_num, void **args_base, void **args, kmp_int64 *arg_sizes,
+ kmp_int64 *arg_types, void **arg_names, void **arg_mappers,
+ kmp_target_data_relocate_t reloc);
#endif
/* Interface to fast scalable reduce methods routines */
diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index 9898fdea5aa338..f6b7e49a1671a2 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -908,7 +908,7 @@ void __kmp_taskgraph_release_locks(kmp_int32 gtid,
/// otherwise. Return number of mutex dependencies in *N_MTXS.
static bool __kmp_filter_aliased_deps(kmp_int32 ndeps,
kmp_depend_info_t *dep_list,
- kmp_task_t *task, int *n_mtxs) {
+ bool can_hold_mutexes, int *n_mtxs) {
*n_mtxs = 0;
// Filter deps in dep_list
@@ -931,7 +931,7 @@ static bool __kmp_filter_aliased_deps(kmp_int32 ndeps,
}
if (dep_list[i].flag == KMP_DEP_MTX) {
// limit number of mtx deps to MAX_MTX_DEPS per node
- if (*n_mtxs < MAX_MTX_DEPS && task != NULL) {
+ if (*n_mtxs < MAX_MTX_DEPS && can_hold_mutexes) {
++(*n_mtxs);
} else {
dep_list[i].flag = KMP_DEP_OUT; // downgrade mutexinoutset to inout
@@ -1371,9 +1371,15 @@ static void __kmp_taskgraph_region_doms(kmp_taskgraph_region_t **order,
}
static bool __kmp_taskgraph_region_mutex_p(kmp_taskgraph_region_t *reg) {
- if (reg->type == TASKGRAPH_REGION_NODE)
+ switch (reg->type) {
+ case TASKGRAPH_REGION_NODE:
+ KMP_TASKGRAPH_REGION_TARGET_CASES:
+ // Only a leaf holds locks of its own; a container's set is the union it
+ // gathers from these, and says nothing about exclusion between siblings.
return reg->mutexset != nullptr;
- return false;
+ default:
+ return false;
+ }
}
// This function collapses graph regions with forms like this:
@@ -2418,9 +2424,10 @@ static void __kmp_taskgraph_gather_mutex_sets(kmp_info_t *thread,
case TASKGRAPH_REGION_ENTRY:
case TASKGRAPH_REGION_EXIT:
case TASKGRAPH_REGION_WAIT:
+ KMP_TASKGRAPH_REGION_TARGET_CASES:
case TASKGRAPH_REGION_NODE:
- // A task node already carries its recorded set membership, and has no
- // children to recurse into.
+ // A leaf already carries the set membership recorded for its node, whether
+ // it is a task or a target, and none of these has children to recurse into.
return;
// Every kind of container folds the same way: what its members hold between
// them does not depend on how it orders them, only on which of them there
@@ -2472,6 +2479,7 @@ static void __kmp_taskgraph_find_exclusive_regions(
case TASKGRAPH_REGION_EXIT:
case TASKGRAPH_REGION_NODE:
case TASKGRAPH_REGION_WAIT:
+ KMP_TASKGRAPH_REGION_TARGET_CASES:
break;
case TASKGRAPH_REGION_SEQUENTIAL:
case TASKGRAPH_REGION_PARALLEL:
@@ -2649,6 +2657,7 @@ __kmp_taskgraph_strip_mutex_sets(kmp_info_t *thread,
case TASKGRAPH_REGION_WAIT:
assert(!region->mutexset);
break;
+ KMP_TASKGRAPH_REGION_TARGET_CASES:
case TASKGRAPH_REGION_NODE:
if (region->mutexset) {
if (in_exclusive) {
@@ -2721,6 +2730,14 @@ __kmp_taskgraph_region_type_name(kmp_taskgraph_region_type type) {
return "node";
case TASKGRAPH_REGION_WAIT:
return "wait";
+ case TASKGRAPH_REGION_TARGET:
+ return "target";
+ case TASKGRAPH_REGION_TARGET_ENTER_DATA:
+ return "target enter data";
+ case TASKGRAPH_REGION_TARGET_EXIT_DATA:
+ return "target exit data";
+ case TASKGRAPH_REGION_TARGET_UPDATE:
+ return "target update";
case TASKGRAPH_REGION_PARALLEL:
return "parallel";
case TASKGRAPH_REGION_EXCLUSIVE:
@@ -2744,7 +2761,9 @@ static void __kmp_dump_taskgraph_regions(FILE *f,
__kmp_taskgraph_region_type_name(region->type));
break;
case TASKGRAPH_REGION_NODE:
- case TASKGRAPH_REGION_WAIT: {
+ case TASKGRAPH_REGION_WAIT:
+ // clang-format off
+ KMP_TASKGRAPH_REGION_TARGET_CASES: {
char set_membership[40];
if (region->mutexset)
sprintf(set_membership, " [sets: 0x%llx]",
@@ -2756,6 +2775,7 @@ static void __kmp_dump_taskgraph_regions(FILE *f,
set_membership);
break;
}
+ // clang-format on
default: {
char set_membership[40];
if (region->mutexset)
@@ -3095,8 +3115,9 @@ kmp_int32 __kmp_build_taskgraph(kmp_int32 gtid,
int n_mtxs = 0;
bool dep_all;
- dep_all = __kmp_filter_aliased_deps(nodes[i].ndeps, nodes[i].dep_list,
- nodes[i].task, &n_mtxs);
+ dep_all = __kmp_filter_aliased_deps(
+ nodes[i].ndeps, nodes[i].dep_list,
+ nodes[i].task != nullptr || nodes[i].target != nullptr, &n_mtxs);
kmp_depnode_t *node = &all_depnodes[i];
__kmp_init_node(node, /*on_stack=*/false);
node->dn.task = nodes[i].task;
@@ -3123,8 +3144,12 @@ kmp_int32 __kmp_build_taskgraph(kmp_int32 gtid,
kmp_taskgraph_region_t *cfg_barrier = nullptr;
for (kmp_int32 i = 0; i < numnodes; i++) {
+ // Target nodes have a null `task` (they would otherwise look like a
+ // taskwait); their region type comes from the recorded target kind.
initial_regions[i].type =
- nodes[i].task ? TASKGRAPH_REGION_NODE : TASKGRAPH_REGION_WAIT;
+ nodes[i].target
+ ? nodes[i].target->kind
+ : (nodes[i].task ? TASKGRAPH_REGION_NODE : TASKGRAPH_REGION_WAIT);
initial_regions[i].task.node = &nodes[i];
initial_regions[i].parent = nullptr;
if (i < numnodes - 1) {
@@ -3172,7 +3197,9 @@ kmp_int32 __kmp_build_taskgraph(kmp_int32 gtid,
initial_regions[i].predecessors);
}
// If we find a post-taskloop WAIT or an undeferred task, record it as a
- // barrier.
+ // barrier. For the undeferred case, this is what makes e.g. "target enter
+ // data", "target", "target exit data" execute in sequence within a
+ // taskgraph.
if (is_taskgroup_marker || nodes[i].undeferred)
cfg_barrier = &initial_regions[i];
}
@@ -3341,7 +3368,7 @@ static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
"dep_barrier=%d .\n",
gtid, taskdata, ndeps, ndeps_noalias, dep_barrier));
- dep_all = __kmp_filter_aliased_deps(ndeps, dep_list, task, &n_mtxs);
+ dep_all = __kmp_filter_aliased_deps(ndeps, dep_list, task != NULL, &n_mtxs);
// doesn't need to be atomic as no other thread is going to be accessing this
// node just yet.
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 59786c4ff82a8c..b9726894789acd 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -26,8 +26,69 @@
#if ENABLE_LIBOMPTARGET
static void (*tgt_target_nowait_query)(void **);
+// libomptarget entry points through which the __kmpc_taskgraph_target* entry
+// points issue their constructs.
+// The kernel-arguments struct (\c __tgt_kernel_arguments) and the map names
+// array (\c map_var_info_t) are kept opaque here (\c void * / \c void **);
+// libomptarget reinterprets them.
+static int (*tgt_target_kernel)(ident_t *, int64_t, int32_t, int32_t, void *,
+ void *);
+static void (*tgt_target_data_begin_mapper)(ident_t *, int64_t, int32_t,
+ void **, void **, int64_t *,
+ int64_t *, void **, void **);
+static void (*tgt_target_data_end_mapper)(ident_t *, int64_t, int32_t, void **,
+ void **, int64_t *, int64_t *,
+ void **, void **);
+static void (*tgt_target_data_update_mapper)(ident_t *, int64_t, int32_t,
+ void **, void **, int64_t *,
+ int64_t *, void **, void **);
+// The 'nowait' map variants take four extra dependence arguments; the
+// __kmpc_taskgraph_target* entry points satisfy dependences themselves, so
+// they are always passed as empty.
+static void (*tgt_target_data_begin_nowait_mapper)(ident_t *, int64_t, int32_t,
+ void **, void **, int64_t *,
+ int64_t *, void **, void **,
+ int32_t, void *, int32_t,
+ void *);
+static void (*tgt_target_data_end_nowait_mapper)(ident_t *, int64_t, int32_t,
+ void **, void **, int64_t *,
+ int64_t *, void **, void **,
+ int32_t, void *, int32_t,
+ void *);
+static void (*tgt_target_data_update_nowait_mapper)(ident_t *, int64_t, int32_t,
+ void **, void **, int64_t *,
+ int64_t *, void **, void **,
+ int32_t, void *, int32_t,
+ void *);
+
+// Deep-copies a kernel-arguments struct and the arrays it points at into a
+// caller-provided block, sizing that block when the destination is null (see
+// offload/include/omptarget.h).
+static void (*tgt_taskgraph_dup_kernel_args)(void *, void *, size_t *);
+
+static void (*tgt_taskgraph_dup_data_args)(void *, int32_t, void ***, void ***,
+ int64_t **, int64_t **, void ***,
+ void ***, size_t *);
+
void __kmp_init_target_task() {
*(void **)(&tgt_target_nowait_query) = KMP_DLSYM("__tgt_target_nowait_query");
+ *(void **)(&tgt_target_kernel) = KMP_DLSYM("__tgt_target_kernel");
+ *(void **)(&tgt_target_data_begin_mapper) =
+ KMP_DLSYM("__tgt_target_data_begin_mapper");
+ *(void **)(&tgt_target_data_end_mapper) =
+ KMP_DLSYM("__tgt_target_data_end_mapper");
+ *(void **)(&tgt_target_data_update_mapper) =
+ KMP_DLSYM("__tgt_target_data_update_mapper");
+ *(void **)(&tgt_target_data_begin_nowait_mapper) =
+ KMP_DLSYM("__tgt_target_data_begin_nowait_mapper");
+ *(void **)(&tgt_target_data_end_nowait_mapper) =
+ KMP_DLSYM("__tgt_target_data_end_nowait_mapper");
+ *(void **)(&tgt_target_data_update_nowait_mapper) =
+ KMP_DLSYM("__tgt_target_data_update_nowait_mapper");
+ *(void **)(&tgt_taskgraph_dup_kernel_args) =
+ KMP_DLSYM("__tgt_taskgraph_dup_kernel_args");
+ *(void **)(&tgt_taskgraph_dup_data_args) =
+ KMP_DLSYM("__tgt_taskgraph_dup_data_args");
}
#endif
@@ -2184,6 +2245,7 @@ static kmp_int32 __kmp_count_region_exits(kmp_taskgraph_region_t *region) {
case TASKGRAPH_REGION_WAIT:
case TASKGRAPH_REGION_ENTRY:
case TASKGRAPH_REGION_EXIT:
+ KMP_TASKGRAPH_REGION_TARGET_CASES:
return 1;
case TASKGRAPH_REGION_PARALLEL: {
// A region with reductions is left through its join descr.
@@ -2221,13 +2283,16 @@ kmp_int32 __kmp_count_exec_descrs_1(kmp_taskgraph_region_t *region,
case TASKGRAPH_REGION_NODE:
case TASKGRAPH_REGION_WAIT:
case TASKGRAPH_REGION_ENTRY:
- case TASKGRAPH_REGION_EXIT: {
+ case TASKGRAPH_REGION_EXIT:
+ // clang-format off
+ KMP_TASKGRAPH_REGION_TARGET_CASES: {
descrs++;
if (npreds > 0) {
successors += npreds;
}
return 1;
}
+ // clang-format on
case TASKGRAPH_REGION_PARALLEL: {
if (npreds > 0)
successors += npreds;
@@ -2472,7 +2537,9 @@ kmp_taskgraph_exec_descr_list_t *__kmp_build_exec_descrs_1(
case TASKGRAPH_REGION_NODE:
case TASKGRAPH_REGION_WAIT:
case TASKGRAPH_REGION_ENTRY:
- case TASKGRAPH_REGION_EXIT: {
+ case TASKGRAPH_REGION_EXIT:
+ // clang-format off
+ KMP_TASKGRAPH_REGION_TARGET_CASES: {
kmp_taskgraph_exec_descr_t *exec_descr =
descr_for_region(region, __kmp_exec_descr_list_len(preds));
// Edit the taskdata for this specific instantiation. At present the
@@ -2490,6 +2557,7 @@ kmp_taskgraph_exec_descr_list_t *__kmp_build_exec_descrs_1(
__kmp_exec_descr_list_deref(thread, recycled, preds);
return __kmp_exec_descr_singleton_list(thread, recycled, exec_descr);
}
+ // clang-format on
case TASKGRAPH_REGION_PARALLEL: {
kmp_int32 npreds = __kmp_exec_descr_list_len(preds);
kmp_taskgraph_exec_descr_t *gather_descr = descr_for_region(region, npreds);
@@ -2692,6 +2760,79 @@ struct kmp_taskred_input;
template <typename T>
void *__kmp_task_reduction_init(int gtid, int num, T *data);
+#if ENABLE_LIBOMPTARGET
+/// Relocate one recorded target construct against TASKGRAPH_ARGS: rewrite the
+/// host addresses of mappings that originate on the stack, and refresh its
+/// by-value firstprivates. What the helper is handed depends on the region
+/// type: the kernel-arguments blob for a kernel construct, the map arrays for
+/// a data construct. The mask it returns is not needed: the host path
+/// re-issues from the refreshed arguments directly.
+static void __kmp_taskgraph_relocate_target(kmp_taskgraph_target_node_t *target,
+ void *taskgraph_args) {
+ switch (target->kind) {
+ case TASKGRAPH_REGION_TARGET:
+ if (target->u.kernel.relocate)
+ target->u.kernel.relocate(target->u.kernel.kernel_args, taskgraph_args);
+ break;
+ default:
+ if (target->u.data.relocate)
+ target->u.data.relocate(target->u.data.args_base, target->u.data.args,
+ taskgraph_args);
+ break;
+ }
+}
+
+/// Re-issue one recorded target construct synchronously.
+///
+/// This is the host replay path: the exec_descr walk reaches a recorded target
+/// region and issues it here. The construct's arguments were relocated when
+/// the replay started (__kmp_taskgraph_relocate_nodes).
+static void __kmp_taskgraph_exec_target(kmp_info_t *thread,
+ kmp_taskgraph_region_t *region) {
+ kmp_taskgraph_target_node_t *target = region->task.node->target;
+ KMP_DEBUG_ASSERT(target);
+
+ switch (target->kind) {
+ case TASKGRAPH_REGION_TARGET:
+ KMP_ASSERT(tgt_target_kernel);
+ tgt_target_kernel(/*loc_ref=*/nullptr, target->device_id,
+ target->u.kernel.num_teams, target->u.kernel.thread_limit,
+ target->u.kernel.host_ptr, target->u.kernel.kernel_args);
+ break;
+ case TASKGRAPH_REGION_TARGET_ENTER_DATA:
+ KMP_ASSERT(tgt_target_data_begin_mapper);
+ tgt_target_data_begin_mapper(
+ /*loc_ref=*/nullptr, target->device_id, target->u.data.arg_num,
+ target->u.data.args_base, target->u.data.args,
+ (int64_t *)target->u.data.arg_sizes,
+ (int64_t *)target->u.data.arg_types, target->u.data.arg_names,
+ target->u.data.arg_mappers);
+ break;
+ case TASKGRAPH_REGION_TARGET_EXIT_DATA:
+ KMP_ASSERT(tgt_target_data_end_mapper);
+ tgt_target_data_end_mapper(
+ /*loc_ref=*/nullptr, target->device_id, target->u.data.arg_num,
+ target->u.data.args_base, target->u.data.args,
+ (int64_t *)target->u.data.arg_sizes,
+ (int64_t *)target->u.data.arg_types, target->u.data.arg_names,
+ target->u.data.arg_mappers);
+ break;
+ case TASKGRAPH_REGION_TARGET_UPDATE:
+ KMP_ASSERT(tgt_target_data_update_mapper);
+ tgt_target_data_update_mapper(
+ /*loc_ref=*/nullptr, target->device_id, target->u.data.arg_num,
+ target->u.data.args_base, target->u.data.args,
+ (int64_t *)target->u.data.arg_sizes,
+ (int64_t *)target->u.data.arg_types, target->u.data.arg_names,
+ target->u.data.arg_mappers);
+ break;
+ default:
+ KMP_DEBUG_ASSERT(false && "not a recorded target region");
+ break;
+ }
+}
+#endif
+
// Decrement DESCR's predecessor count; if it reaches zero, fire it. TASKGROUP
// is the taskgroup any task spawned by DESCR should join.
static void __kmp_taskgraph_exec_descr_start(kmp_int32 gtid, kmp_info_t *thread,
@@ -2740,6 +2881,27 @@ static void __kmp_taskgraph_exec_descr_start(kmp_int32 gtid, kmp_info_t *thread,
for (kmp_taskgraph_exec_descr_elem_t *s = descr->successors; s; s = s->next)
__kmp_taskgraph_exec_descr_start(gtid, thread, s->exec_descr, taskgroup);
break;
+ // clang-format off
+ KMP_TASKGRAPH_REGION_TARGET_CASES: {
+ // Replay a recorded target construct inline. The libomptarget entry
+ // points used here are synchronous, so the successors are released as soon
+ // as the call returns -- as for a WAIT region, rather than from the
+ // task-completion hook a NODE region relies on.
+#if ENABLE_LIBOMPTARGET
+ if (descr->region->mutexset)
+ __kmp_taskgraph_acquire_locks(gtid, descr->region->owner, descr->region);
+ __kmp_taskgraph_exec_target(thread, descr->region);
+ if (descr->region->mutexset)
+ __kmp_taskgraph_release_locks(gtid, descr->region->owner, descr->region);
+ for (kmp_taskgraph_exec_descr_elem_t *s = descr->successors; s; s = s->next)
+ __kmp_taskgraph_exec_descr_start(gtid, thread, s->exec_descr, taskgroup);
+#else
+ // A target region cannot have been recorded without libomptarget.
+ abort();
+#endif
+ break;
+ }
+ // clang-format on
case TASKGRAPH_REGION_EXCLUSIVE:
case TASKGRAPH_REGION_SEQUENTIAL:
case TASKGRAPH_REGION_ENTRY:
@@ -3457,15 +3619,15 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
}
#if OMP_TASKGRAPH_EXPERIMENTAL
-/// Relocate the recorded tasks of TASKGRAPH against this replay's
-/// taskgraph_args.
+/// Relocate the recorded nodes of TASKGRAPH against this replay's
+/// taskgraph_args, host tasks and target constructs alike.
///
/// All of it happens here, before the replay releases any task, and none of
-/// it as a task is dispatched. A replay re-encounters every recorded
+/// it as a node is dispatched. A replay re-encounters every recorded
/// construct up front (OpenMP 6.0 [14.3]), so a firstprivate takes the value
/// its variable has when the replay starts, and a list item's base pointer is
/// re-read at the same point. Doing it at dispatch would instead observe
-/// whatever the task's predecessors in the graph had written by then.
+/// whatever the node's predecessors in the graph had written by then.
///
/// FIXME: That is too early for a construct after a 'taskwait' with a 'depend'
/// clause, which a replay should only encounter once the tasks the 'taskwait'
@@ -3474,6 +3636,12 @@ static void __kmp_taskgraph_relocate_nodes(kmp_taskgraph_record_t *taskgraph) {
void *taskgraph_args = taskgraph->taskgraph_args;
for (kmp_int32 i = 0; i < taskgraph->num_tasks; i++) {
kmp_taskgraph_node_t *node = &taskgraph->record_map[i];
+ if (node->target) {
+#if ENABLE_LIBOMPTARGET
+ __kmp_taskgraph_relocate_target(node->target, taskgraph_args);
+#endif
+ continue;
+ }
if (!node->task)
continue;
if (node->relocate) {
@@ -5368,6 +5536,7 @@ __kmp_taskgraph_node_alloc(kmp_taskgraph_record_t *rec, kmp_task_t *task,
new_task->undeferred = false;
new_task->relocate = nullptr;
new_task->reduce_input = nullptr;
+ new_task->target = nullptr;
new_task->ndeps = 0;
new_task->dep_list = nullptr;
new_task->cfg_successor = -1;
@@ -6158,6 +6327,9 @@ __kmp_taskgraph_free_region_metadata(kmp_info_t *thread,
case TASKGRAPH_REGION_EXIT:
case TASKGRAPH_REGION_WAIT:
case TASKGRAPH_REGION_NODE:
+ // Target payloads are owned by the node (freed in __kmp_taskgraph_free), not
+ // by the region; nothing region-level to free here.
+ KMP_TASKGRAPH_REGION_TARGET_CASES:
break;
case TASKGRAPH_REGION_PARALLEL:
case TASKGRAPH_REGION_SEQUENTIAL:
@@ -6174,6 +6346,32 @@ __kmp_taskgraph_free_region_metadata(kmp_info_t *thread,
}
}
+// Free the deep-copied capture owned by a target node (allocated during
+// recording in the __kmpc_taskgraph_target* entry points).
+static void
+__kmp_taskgraph_free_target_node(kmp_info_t *thread,
+ kmp_taskgraph_target_node_t *target) {
+ switch (target->kind) {
+ case TASKGRAPH_REGION_TARGET:
+ // One block holding the blob and every array it points at (the arrays were
+ // repointed into it by __tgt_taskgraph_dup_kernel_args at record time).
+ if (target->u.kernel.kernel_args)
+ __kmp_thread_free(thread, target->u.kernel.kernel_args);
+ break;
+ case TASKGRAPH_REGION_TARGET_ENTER_DATA:
+ case TASKGRAPH_REGION_TARGET_EXIT_DATA:
+ case TASKGRAPH_REGION_TARGET_UPDATE:
+ // One block holding every array, laid out by
+ // __tgt_taskgraph_dup_data_args at record time.
+ assert(target->u.data.arena);
+ __kmp_thread_free(thread, target->u.data.arena);
+ break;
+ default:
+ break;
+ }
+ __kmp_thread_free(thread, target);
+}
+
static void __kmp_taskgraph_free(kmp_int32 gtid, kmp_taskgraph_record_t *rec,
bool keep_rec = false) {
kmp_info_t *thread = __kmp_threads[gtid];
@@ -6192,6 +6390,10 @@ static void __kmp_taskgraph_free(kmp_int32 gtid, kmp_taskgraph_record_t *rec,
thread, rec->root, keep_rec ? &rec->recycled_deps : nullptr);
for (size_t task = 0; task < rec->num_tasks; task++) {
+ // Free any deep-copied target payload first (present on target nodes,
+ // whose 'task' is null so they are otherwise skipped below).
+ if (rec->record_map[task].target)
+ __kmp_taskgraph_free_target_node(thread, rec->record_map[task].target);
// Skip entries that don't have an associated task (e.g. taskwait nodes
// recorded by __kmpc_taskgraph_taskwait).
if (rec->record_map[task].task == nullptr)
@@ -6578,6 +6780,245 @@ void __kmpc_taskgraph_taskwait(ident_t *loc_ref, kmp_int32 gtid,
has_no_wait);
}
+#if ENABLE_LIBOMPTARGET
+// Allocate a target node, copy its depend list, and attach a fresh target
+// payload of the given kind. Shared by the kernel and data-op recorders.
+static kmp_taskgraph_target_node_t *__kmp_taskgraph_alloc_target_node(
+ kmp_info_t *thread, kmp_taskgraph_record_t *rec,
+ enum kmp_taskgraph_region_type kind, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_int32 has_no_wait, kmp_int64 device_id) {
+ kmp_taskgraph_node_t *node = __kmp_taskgraph_node_alloc(rec, nullptr);
+ node->undeferred = !has_no_wait;
+ node->ndeps = ndeps;
+ node->dep_list = (kmp_depend_info_t *)__kmp_thread_malloc(
+ thread, ndeps * sizeof(kmp_depend_info_t));
+ if (ndeps)
+ KMP_MEMCPY(node->dep_list, dep_list, ndeps * sizeof(kmp_depend_info_t));
+
+ kmp_taskgraph_target_node_t *target =
+ (kmp_taskgraph_target_node_t *)__kmp_thread_malloc(
+ thread, sizeof(kmp_taskgraph_target_node_t));
+ target->kind = kind;
+ target->device_id = device_id;
+ node->target = target;
+ return target;
+}
+
+// Record a target kernel construct, deep-copying the kernel-arguments blob and
+// each array it points to so the capture outlives this recording invocation.
+static void __kmp_taskgraph_record_target_kernel(
+ kmp_info_t *thread, kmp_taskgraph_record_t *rec, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_int32 has_no_wait, kmp_int64 device_id,
+ kmp_int32 num_teams, kmp_int32 thread_limit, void *host_ptr,
+ void *kernel_args, kmp_target_kernel_relocate_t relocate) {
+ kmp_taskgraph_target_node_t *target = __kmp_taskgraph_alloc_target_node(
+ thread, rec, TASKGRAPH_REGION_TARGET, ndeps, dep_list, has_no_wait,
+ device_id);
+ target->u.kernel.relocate = relocate;
+ target->u.kernel.num_teams = num_teams;
+ target->u.kernel.thread_limit = thread_limit;
+ target->u.kernel.host_ptr = host_ptr;
+
+ size_t alloc_size = 0;
+ // We call libomptarget/tgt_taskgraph_dup_kernel_args twice: the first time
+ // to size the block we need to allocate, and the second time to copy the
+ // kernel arguments. This library doesn't need to be concerned with the
+ // exact bit layout of these types: they are consumed by libomptarget anyway.
+ tgt_taskgraph_dup_kernel_args(nullptr, kernel_args, &alloc_size);
+ void *dup_kernel_args = __kmp_thread_malloc(thread, alloc_size);
+ tgt_taskgraph_dup_kernel_args(dup_kernel_args, kernel_args, &alloc_size);
+
+ target->u.kernel.kernel_args = dup_kernel_args;
+}
+
+// Copy ARG_NUM elements of ELEM_SIZE bytes from SRC into the arena slice at
+// BUMP_PTR, advancing it past the slice. A null SRC yields a null result: the
+// optional map arrays (arg_names, arg_mappers) have to stay absent rather than
+// becoming pointers to uninitialized storage, because libomptarget tests them
+// for null.
+[[maybe_unused]] static void *__kmp_taskgraph_bump_dup(char *&bump_ptr,
+ const void *src,
+ kmp_int32 arg_num,
+ size_t elem_size) {
+ char *dest = bump_ptr;
+ bump_ptr += arg_num * elem_size;
+ if (!src)
+ return nullptr;
+ KMP_MEMCPY(dest, src, arg_num * elem_size);
+ return dest;
+}
+
+// Record a target data construct (enter/exit/update), deep-copying the six map
+// arrays into a single block so the capture outlives this recording invocation.
+//
+// libomptarget does the copying, as it does for a kernel node: an element of
+// these arrays can be a pointer to something that has to be copied too, and
+// which one is a question about map types that belongs on the side that
+// defines them.
+static void __kmp_taskgraph_record_target_data(
+ kmp_info_t *thread, kmp_taskgraph_record_t *rec,
+ enum kmp_taskgraph_region_type kind, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_int32 has_no_wait, kmp_int64 device_id,
+ kmp_int32 arg_num, void **args_base, void **args, kmp_int64 *arg_sizes,
+ kmp_int64 *arg_types, void **arg_names, void **arg_mappers,
+ kmp_target_data_relocate_t relocate) {
+ kmp_taskgraph_target_node_t *target = __kmp_taskgraph_alloc_target_node(
+ thread, rec, kind, ndeps, dep_list, has_no_wait, device_id);
+ target->u.data.relocate = relocate;
+ target->u.data.arg_num = arg_num;
+ // Both calls are given the originals; the second repoints these at the copy.
+ void **dup_args_base = args_base;
+ void **dup_args = args;
+ int64_t *dup_arg_sizes = (int64_t *)arg_sizes;
+ int64_t *dup_arg_types = (int64_t *)arg_types;
+ void **dup_arg_names = arg_names;
+ void **dup_arg_mappers = arg_mappers;
+ size_t alloc_size = 0;
+ tgt_taskgraph_dup_data_args(nullptr, arg_num, &dup_args_base, &dup_args,
+ &dup_arg_sizes, &dup_arg_types, &dup_arg_names,
+ &dup_arg_mappers, &alloc_size);
+ void *arena = __kmp_thread_malloc(thread, alloc_size);
+ tgt_taskgraph_dup_data_args(arena, arg_num, &dup_args_base, &dup_args,
+ &dup_arg_sizes, &dup_arg_types, &dup_arg_names,
+ &dup_arg_mappers, &alloc_size);
+ target->u.data.arena = arena;
+ target->u.data.args_base = dup_args_base;
+ target->u.data.args = dup_args;
+ target->u.data.arg_sizes = (kmp_int64 *)dup_arg_sizes;
+ target->u.data.arg_types = (kmp_int64 *)dup_arg_types;
+ target->u.data.arg_names = dup_arg_names;
+ target->u.data.arg_mappers = dup_arg_mappers;
+}
+
+#endif
+
+// Entry points for target constructs that should be recorded into a taskgraph,
+// either because they are lexically nested in one or because they carry a
+// `replayable` clause.
+
+kmp_int32 __kmpc_taskgraph_target(ident_t *loc_ref, kmp_int32 gtid,
+ kmp_int32 ndeps, kmp_depend_info_t *dep_list,
+ kmp_int32 has_no_wait, kmp_int64 device_id,
+ kmp_int32 num_teams, kmp_int32 thread_limit,
+ void *host_ptr, void *kernel_args,
+ kmp_target_kernel_relocate_t reloc) {
+ if (ndeps > 0)
+ __kmpc_omp_taskwait_deps_51(loc_ref, gtid, ndeps, dep_list, 0, nullptr,
+ /*has_no_wait=*/0);
+#if ENABLE_LIBOMPTARGET
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgraph_record_t *rec =
+ __kmp_taskgraph_task_recording(thread->th.th_current_task);
+ if (rec && KMP_ATOMIC_LD_ACQ(&rec->status) == KMP_TDG_RECORDING)
+ __kmp_taskgraph_record_target_kernel(
+ thread, rec, ndeps, dep_list, has_no_wait, device_id, num_teams,
+ thread_limit, host_ptr, kernel_args, reloc);
+ KMP_ASSERT(tgt_target_kernel);
+ // Recording or not, the region is launched synchronously, once the
+ // dependences above have been satisfied.
+ return tgt_target_kernel(loc_ref, device_id, num_teams, thread_limit,
+ host_ptr, kernel_args);
+#else
+ return 0;
+#endif
+}
+
+void __kmpc_taskgraph_target_enter_data(
+ ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_int32 has_no_wait, kmp_int64 device_id,
+ kmp_int32 arg_num, void **args_base, void **args, kmp_int64 *arg_sizes,
+ kmp_int64 *arg_types, void **arg_names, void **arg_mappers,
+ kmp_target_data_relocate_t reloc) {
+ if (ndeps > 0)
+ __kmpc_omp_taskwait_deps_51(loc_ref, gtid, ndeps, dep_list, 0, nullptr,
+ /*has_no_wait=*/0);
+#if ENABLE_LIBOMPTARGET
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgraph_record_t *rec =
+ __kmp_taskgraph_task_recording(thread->th.th_current_task);
+ bool recording = rec && KMP_ATOMIC_LD_ACQ(&rec->status) == KMP_TDG_RECORDING;
+ if (recording) {
+ __kmp_taskgraph_record_target_data(
+ thread, rec, TASKGRAPH_REGION_TARGET_ENTER_DATA, ndeps, dep_list,
+ has_no_wait, device_id, arg_num, args_base, args, arg_sizes, arg_types,
+ arg_names, arg_mappers, reloc);
+ } else if (has_no_wait && tgt_target_data_begin_nowait_mapper) {
+ tgt_target_data_begin_nowait_mapper(
+ loc_ref, device_id, arg_num, args_base, args, (int64_t *)arg_sizes,
+ (int64_t *)arg_types, arg_names, arg_mappers, 0, nullptr, 0, nullptr);
+ return;
+ }
+ KMP_ASSERT(tgt_target_data_begin_mapper);
+ tgt_target_data_begin_mapper(loc_ref, device_id, arg_num, args_base, args,
+ (int64_t *)arg_sizes, (int64_t *)arg_types,
+ arg_names, arg_mappers);
+#endif
+}
+
+void __kmpc_taskgraph_target_exit_data(
+ ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_int32 has_no_wait, kmp_int64 device_id,
+ kmp_int32 arg_num, void **args_base, void **args, kmp_int64 *arg_sizes,
+ kmp_int64 *arg_types, void **arg_names, void **arg_mappers,
+ kmp_target_data_relocate_t reloc) {
+ if (ndeps > 0)
+ __kmpc_omp_taskwait_deps_51(loc_ref, gtid, ndeps, dep_list, 0, nullptr,
+ /*has_no_wait=*/0);
+#if ENABLE_LIBOMPTARGET
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgraph_record_t *rec =
+ __kmp_taskgraph_task_recording(thread->th.th_current_task);
+ bool recording = rec && KMP_ATOMIC_LD_ACQ(&rec->status) == KMP_TDG_RECORDING;
+ if (recording) {
+ __kmp_taskgraph_record_target_data(
+ thread, rec, TASKGRAPH_REGION_TARGET_EXIT_DATA, ndeps, dep_list,
+ has_no_wait, device_id, arg_num, args_base, args, arg_sizes, arg_types,
+ arg_names, arg_mappers, reloc);
+ } else if (has_no_wait && tgt_target_data_end_nowait_mapper) {
+ tgt_target_data_end_nowait_mapper(
+ loc_ref, device_id, arg_num, args_base, args, (int64_t *)arg_sizes,
+ (int64_t *)arg_types, arg_names, arg_mappers, 0, nullptr, 0, nullptr);
+ return;
+ }
+ KMP_ASSERT(tgt_target_data_end_mapper);
+ tgt_target_data_end_mapper(loc_ref, device_id, arg_num, args_base, args,
+ (int64_t *)arg_sizes, (int64_t *)arg_types,
+ arg_names, arg_mappers);
+#endif
+}
+
+void __kmpc_taskgraph_target_update(
+ ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
+ kmp_depend_info_t *dep_list, kmp_int32 has_no_wait, kmp_int64 device_id,
+ kmp_int32 arg_num, void **args_base, void **args, kmp_int64 *arg_sizes,
+ kmp_int64 *arg_types, void **arg_names, void **arg_mappers,
+ kmp_target_data_relocate_t reloc) {
+ if (ndeps > 0)
+ __kmpc_omp_taskwait_deps_51(loc_ref, gtid, ndeps, dep_list, 0, nullptr,
+ /*has_no_wait=*/0);
+#if ENABLE_LIBOMPTARGET
+ kmp_info_t *thread = __kmp_threads[gtid];
+ kmp_taskgraph_record_t *rec =
+ __kmp_taskgraph_task_recording(thread->th.th_current_task);
+ bool recording = rec && KMP_ATOMIC_LD_ACQ(&rec->status) == KMP_TDG_RECORDING;
+ if (recording) {
+ __kmp_taskgraph_record_target_data(
+ thread, rec, TASKGRAPH_REGION_TARGET_UPDATE, ndeps, dep_list,
+ has_no_wait, device_id, arg_num, args_base, args, arg_sizes, arg_types,
+ arg_names, arg_mappers, reloc);
+ } else if (has_no_wait && tgt_target_data_update_nowait_mapper) {
+ tgt_target_data_update_nowait_mapper(
+ loc_ref, device_id, arg_num, args_base, args, (int64_t *)arg_sizes,
+ (int64_t *)arg_types, arg_names, arg_mappers, 0, nullptr, 0, nullptr);
+ return;
+ }
+ KMP_ASSERT(tgt_target_data_update_mapper);
+ tgt_target_data_update_mapper(loc_ref, device_id, arg_num, args_base, args,
+ (int64_t *)arg_sizes, (int64_t *)arg_types,
+ arg_names, arg_mappers);
+#endif
+}
+
kmp_uint32 __kmpc_taskgraph_taskloop(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
kmp_int32 if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
>From 5ce9c54629b645079d6f6532d6c258cf1acb99b3 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 17 Sep 2026 05:55:21 -0500
Subject: [PATCH 14/29] [Offload] Support LIBOMP_TASKGRAPH_EXPERIMENTAL for
offload tests, plus replayable(false) test
Taskgraph support in the host runtime -- the __kmpc_taskgraph*
entry points -- is experimental, and only built with the
LIBOMP_TASKGRAPH_EXPERIMENTAL option. This patch passes the option down
to libomptarget's testsuite.
taskgraph_replayable_false.c, the first test to use it, checks
'replayable(false)' on the target family inside a taskgraph region.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/225712
---
offload/test/CMakeLists.txt | 4 +
offload/test/lit.cfg | 6 +
offload/test/lit.site.cfg.in | 1 +
.../offloading/taskgraph_replayable_false.c | 109 ++++++++++++++++++
4 files changed, 120 insertions(+)
create mode 100644 offload/test/offloading/taskgraph_replayable_false.c
diff --git a/offload/test/CMakeLists.txt b/offload/test/CMakeLists.txt
index be2704cd6dee1a..bdba1bc08b7b79 100644
--- a/offload/test/CMakeLists.txt
+++ b/offload/test/CMakeLists.txt
@@ -12,6 +12,10 @@ else()
set(LIBOMPTARGET_DEBUG False)
endif()
+# The tests of the host runtime's experimental taskgraph support only run where
+# it is built. LIBOMP_* options reach this build along with openmp's.
+pythonize_bool(LIBOMP_TASKGRAPH_EXPERIMENTAL)
+
# Replace the space from user's input with ";" in case that CMake add escape
# char into the lit command.
string(REPLACE " " ";" LIBOMPTARGET_LIT_ARG_LIST "${LIBOMPTARGET_LIT_ARGS}")
diff --git a/offload/test/lit.cfg b/offload/test/lit.cfg
index f042b60d2f1a0c..a5bd889875fcf1 100644
--- a/offload/test/lit.cfg
+++ b/offload/test/lit.cfg
@@ -183,6 +183,12 @@ if has_flang_rt:
if has_libc:
config.available_features.add('libc')
+# Taskgraph (task record & replay) support in the host runtime is experimental,
+# built only with the LIBOMP_TASKGRAPH_EXPERIMENTAL option. The feature has
+# the name libomp's own tests give it.
+if config.has_omp_taskgraph_experimental:
+ config.available_features.add('omp_taskgraph_experimental')
+
# Determine whether the test system supports unified memory.
# For CUDA, this is the case with compute capability 70 (Volta) or higher.
# For all other targets, we currently assume it is.
diff --git a/offload/test/lit.site.cfg.in b/offload/test/lit.site.cfg.in
index 011de60aa2c7f8..0ea93db2ba6cb3 100644
--- a/offload/test/lit.site.cfg.in
+++ b/offload/test/lit.site.cfg.in
@@ -26,6 +26,7 @@ config.libomptarget_not = "@OFFLOAD_NOT_EXECUTABLE@"
config.offload_device_info = "@OFFLOAD_DEVICE_INFO_EXECUTABLE@"
config.libomptarget_debug = @LIBOMPTARGET_DEBUG@
config.has_libomptarget_ompt = @LIBOMPTARGET_OMPT_SUPPORT@
+config.has_omp_taskgraph_experimental = @LIBOMP_TASKGRAPH_EXPERIMENTAL@
config.offload_tblgen = "@OFFLOAD_TBLGEN_EXECUTABLE@"
config.omp_kernel_replay = "@OMP_KERNEL_REPLAY@"
# Let the main config do the real work.
diff --git a/offload/test/offloading/taskgraph_replayable_false.c b/offload/test/offloading/taskgraph_replayable_false.c
new file mode 100644
index 00000000000000..c8334400d03b1e
--- /dev/null
+++ b/offload/test/offloading/taskgraph_replayable_false.c
@@ -0,0 +1,109 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// replayable(false) on the target family inside an 'omp taskgraph' region.
+//
+// OpenMP 6.0 [14.3] makes a construct in a taskgraph construct replayable
+// "unless otherwise specified by the replayable clause", and a replay execution
+// does not execute the region body. So a target construct that opts out is
+// launched by the ordinary libomptarget path on the execution that runs the
+// body, and does not appear in the record -- it is absent from every replay,
+// while its replayable neighbours run on each one.
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 1024
+#define NUM_REPS 4 // one recording execution + three replays
+
+int main() {
+ int *a = (int *)malloc(N * sizeof(int));
+ int *b = (int *)malloc(N * sizeof(int));
+ int *c = (int *)malloc(N * sizeof(int));
+ int *d = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = b[i] = c[i] = d[i] = i;
+
+ // A replayable target next to one that opts out: the first runs on every
+ // execution of the region, the second only on the one that records.
+ for (int rep = 0; rep < NUM_REPS; ++rep) {
+#pragma omp taskgraph graph_id(1)
+ {
+#pragma omp target map(tofrom : a[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] += 1;
+ }
+#pragma omp target map(tofrom : b[0 : N]) replayable(false)
+ {
+ for (int i = 0; i < N; ++i)
+ b[i] += 1;
+ }
+ }
+ }
+
+ // A non-replayable 'target update' reading back a buffer that a replayable
+ // target updates on the device. The enclosing 'target data' maps with 'to',
+ // so nothing is copied back when it ends and the host only ever sees the
+ // value the update-from fetched -- on the recording execution alone.
+#pragma omp target data map(to : c[0 : N])
+ {
+ for (int rep = 0; rep < NUM_REPS; ++rep) {
+#pragma omp taskgraph graph_id(2)
+ {
+#pragma omp target map(alloc : c[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ c[i] += 1;
+ }
+#pragma omp target update from(c[0 : N]) replayable(false)
+ }
+ }
+ }
+
+ // A whole map / compute / read-back chain that opts out: all four constructs
+ // run on the recording execution and the record stays empty, so no replay
+ // does anything at all.
+ for (int rep = 0; rep < NUM_REPS; ++rep) {
+#pragma omp taskgraph graph_id(3)
+ {
+#pragma omp target enter data map(to : d[0 : N]) replayable(false)
+#pragma omp target map(alloc : d[0 : N]) replayable(false)
+ {
+ for (int i = 0; i < N; ++i)
+ d[i] += 5;
+ }
+#pragma omp target update from(d[0 : N]) replayable(false)
+#pragma omp target exit data map(release : d[0 : N]) replayable(false)
+ }
+ }
+
+ int errors = 0;
+ for (int i = 0; i < N; ++i) {
+ // Replayable: incremented on every execution of the region.
+ if (a[i] != i + NUM_REPS)
+ ++errors;
+ // Not replayable: encountered once, on the recording execution.
+ if (b[i] != i + 1)
+ ++errors;
+ // Device-side increment on every execution, read back on the first only.
+ if (c[i] != i + 1)
+ ++errors;
+ if (d[i] != i + 5)
+ ++errors;
+ }
+
+ // CHECK: taskgraph replayable(false) target errors: 0
+ printf("taskgraph replayable(false) target errors: %d\n", errors);
+ free(a);
+ free(b);
+ free(c);
+ free(d);
+ return errors != 0;
+}
>From e06467e705c7a23aef439ac7592910c2084dea01 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 19 Aug 2026 09:51:40 -0500
Subject: [PATCH 15/29] [Clang][OpenMP][IRBuilder] Front-end parts for
taskgraph target constructs
This patch implements support in Clang for new entry points to use for
recordable "omp target", "omp target {enter,exit} data", "omp target
update" directives (introduced by "Taskgraph target directive recording:
runtime parts"). These entry points bypass the hidden helper tasks used
for "nowait" target directives: instead, dependences are passed to the
new target-specific libomp entry points where they can be recorded for
later replay (in the same way as taskgraph support for host tasks).
The recording entry points take the same __tgt_kernel_arguments struct
as an ordinary launch, so OpenMPIRBuilder::emitKernelArgsStruct is
outlined from emitTargetKernel to populate it for both. That part is
no functional change for ordinary launches.
Tests have been added to cover new functionality.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/225714
---
clang/lib/CodeGen/CGOpenMPRuntime.cpp | 518 +++++++++++++-----
clang/lib/CodeGen/CGOpenMPRuntime.h | 57 +-
clang/lib/CodeGen/CGStmtOpenMP.cpp | 33 +-
.../test/OpenMP/taskgraph_target_codegen.cpp | 53 ++
.../OpenMP/taskgraph_target_data_codegen.cpp | 44 ++
.../llvm/Frontend/OpenMP/OMPIRBuilder.h | 13 +
.../include/llvm/Frontend/OpenMP/OMPKinds.def | 18 +
llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 22 +-
offload/test/offloading/taskgraph_target.c | 41 ++
.../test/offloading/taskgraph_target_data.c | 59 ++
.../offloading/taskgraph_target_irreducible.c | 81 +++
.../taskgraph_target_nowait_schedule.c | 81 +++
.../test/offloading/taskgraph_target_record.c | 63 +++
.../test/offloading/taskgraph_target_replay.c | 47 ++
.../taskgraph_target_replay_mixed.c | 57 ++
.../offloading/taskgraph_target_replayable.c | 97 ++++
16 files changed, 1100 insertions(+), 184 deletions(-)
create mode 100644 clang/test/OpenMP/taskgraph_target_codegen.cpp
create mode 100644 clang/test/OpenMP/taskgraph_target_data_codegen.cpp
create mode 100644 offload/test/offloading/taskgraph_target.c
create mode 100644 offload/test/offloading/taskgraph_target_data.c
create mode 100644 offload/test/offloading/taskgraph_target_irreducible.c
create mode 100644 offload/test/offloading/taskgraph_target_nowait_schedule.c
create mode 100644 offload/test/offloading/taskgraph_target_record.c
create mode 100644 offload/test/offloading/taskgraph_target_replay.c
create mode 100644 offload/test/offloading/taskgraph_target_replay_mixed.c
create mode 100644 offload/test/offloading/taskgraph_target_replayable.c
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 065b9e47956b94..95191d9acff7df 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -12121,6 +12121,105 @@ emitClauseForBareTargetDirective(CodeGenFunction &CGF,
}
}
+void CGOpenMPRuntime::emitTaskgraphTargetCall(
+ CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc,
+ llvm::Value *OutlinedFnID, llvm::Value *DeviceID, llvm::Value *NumTeams,
+ llvm::Value *NumThreads, llvm::Value *KernelArgsPtr) {
+ llvm::Value *ThreadID = getThreadID(CGF, Loc);
+ llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+
+ OMPTaskDataTy Data;
+ buildDependences(D, Data);
+ Address DependenciesArray = Address::invalid();
+ llvm::Value *NumOfElements;
+ std::tie(NumOfElements, DependenciesArray) =
+ emitDependClause(CGF, Data.Dependences, Loc);
+
+ llvm::Value *NumDeps;
+ llvm::Value *DepList;
+ if (Data.Dependences.empty()) {
+ NumDeps = CGF.Builder.getInt32(0);
+ DepList = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ } else {
+ NumDeps = NumOfElements;
+ DepList = DependenciesArray.emitRawPointer(CGF);
+ }
+
+ llvm::Value *HasNoWait =
+ CGF.Builder.getInt32(D.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
+
+ // Relocation callback used at replay to rewrite moved host pointers in the
+ // captured kernel arguments. Emission of a real relocate function is not yet
+ // implemented (the recorded graph is not replayed through this path yet), so
+ // a null callback is passed; the runtime simply stores it.
+ llvm::Value *Relocate = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+
+ llvm::Value *Args[] = {UpLoc, ThreadID, NumDeps, DepList,
+ HasNoWait, DeviceID, NumTeams, NumThreads,
+ OutlinedFnID, KernelArgsPtr, Relocate};
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_taskgraph_target),
+ Args);
+}
+
+void CGOpenMPRuntime::emitTaskgraphTargetDataCall(
+ CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc,
+ llvm::Value *DeviceID, unsigned NumTargetItems,
+ llvm::Value *BasePointersArray, llvm::Value *PointersArray,
+ llvm::Value *SizesArray, llvm::Value *MapTypesArray,
+ llvm::Value *MapNamesArray, llvm::Value *MappersArray) {
+ llvm::Value *ThreadID = getThreadID(CGF, Loc);
+ llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+
+ OMPTaskDataTy Data;
+ buildDependences(D, Data);
+ Address DependenciesArray = Address::invalid();
+ llvm::Value *NumOfElements;
+ std::tie(NumOfElements, DependenciesArray) =
+ emitDependClause(CGF, Data.Dependences, Loc);
+
+ llvm::Value *NumDeps;
+ llvm::Value *DepList;
+ if (Data.Dependences.empty()) {
+ NumDeps = CGF.Builder.getInt32(0);
+ DepList = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ } else {
+ NumDeps = NumOfElements;
+ DepList = DependenciesArray.emitRawPointer(CGF);
+ }
+
+ llvm::Value *HasNoWait =
+ CGF.Builder.getInt32(D.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
+ llvm::Value *PointerNum = CGF.Builder.getInt32(NumTargetItems);
+
+ RuntimeFunction RTLFn;
+ switch (D.getDirectiveKind()) {
+ case OMPD_target_enter_data:
+ RTLFn = OMPRTL___kmpc_taskgraph_target_enter_data;
+ break;
+ case OMPD_target_exit_data:
+ RTLFn = OMPRTL___kmpc_taskgraph_target_exit_data;
+ break;
+ case OMPD_target_update:
+ RTLFn = OMPRTL___kmpc_taskgraph_target_update;
+ break;
+ default:
+ llvm_unreachable("Unexpected standalone target data directive.");
+ }
+
+ // See emitTaskgraphTargetCall: a real relocate callback is not emitted yet,
+ // so a null pointer is passed and merely stored by the runtime.
+ llvm::Value *Relocate = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+
+ llvm::Value *Args[] = {UpLoc, ThreadID, NumDeps,
+ DepList, HasNoWait, DeviceID,
+ PointerNum, BasePointersArray, PointersArray,
+ SizesArray, MapTypesArray, MapNamesArray,
+ MappersArray, Relocate};
+ CGF.EmitRuntimeCall(
+ OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), RTLFn), Args);
+}
+
static void emitTargetCallKernelLaunch(
CGOpenMPRuntime *OMPRuntime, llvm::Function *OutlinedFn,
const OMPExecutableDirective &D,
@@ -12132,7 +12231,7 @@ static void emitTargetCallKernelLaunch(
llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
const OMPLoopDirective &D)>
SizeEmitter,
- CodeGenFunction &CGF, CodeGenModule &CGM) {
+ CodeGenFunction &CGF, CodeGenModule &CGM, const Expr *ReplayableCond) {
llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->getOMPBuilder();
// Fill up the arrays with all the captured variables.
@@ -12172,10 +12271,10 @@ static void emitTargetCallKernelLaunch(
MapTypesArray = Info.RTArgs.MapTypesArray;
MapNamesArray = Info.RTArgs.MapNamesArray;
- auto &&ThenGen = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
- RequiresOuterTask, &CS, OffloadingMandatory, Device,
- OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
- SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
+ auto &&EmitBody = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
+ RequiresOuterTask, &CS, OffloadingMandatory, Device,
+ OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
+ SizeEmitter](CodeGenFunction &CGF, bool Recording) {
bool IsReverseOffloading = Device.getInt() == OMPC_DEVICE_ancestor;
if (IsReverseOffloading) {
@@ -12237,17 +12336,79 @@ static void emitTargetCallKernelLaunch(
DynCGroupMem, HasNoWait, /*StrictBlocks=*/IsBare,
/*StrictThreads=*/IsBare, DynCGroupMemFallback);
- llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
- cantFail(OMPRuntime->getOMPBuilder().emitKernelLaunch(
- CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,
- RTLoc, AllocaIP));
- CGF.Builder.restoreIP(AfterIP);
+ if (Recording) {
+ // Record the target region via __kmpc_taskgraph_target instead of
+ // launching it directly. Build the same __tgt_kernel_arguments struct the
+ // kernel launch would use and hand it, together with the launch
+ // parameters and the depend list, to the taskgraph runtime.
+ //
+ // Clear the NoWait flag in the kernel-args struct and drop the original
+ // nowait intent: a replayable target executes synchronously (it is never
+ // wrapped in a hidden helper task), and __kmpc_taskgraph_target launches
+ // it through __tgt_target_kernel synchronously.
+ llvm::OpenMPIRBuilder::TargetKernelArgs TaskgraphArgs = Args;
+ TaskgraphArgs.HasNoWait = false;
+ llvm::SmallVector<llvm::Value *> KernelArgsVector;
+ llvm::OpenMPIRBuilder::getKernelArgsVector(TaskgraphArgs, CGF.Builder,
+ KernelArgsVector);
+ llvm::Value *KernelArgsPtr =
+ OMPRuntime->getOMPBuilder().emitKernelArgsStruct(
+ CGF.Builder, AllocaIP, KernelArgsVector);
+ // emitKernelArgsStruct drives the OMPIRBuilder's own IRBuilder; keep
+ // CGF's builder in sync so the runtime call is emitted after the stores.
+ CGF.Builder.restoreIP(OMPRuntime->getOMPBuilder().getInsertionPoint());
+ // The taskgraph entry point satisfies the dependences itself; there is no
+ // enclosing helper task on this path.
+ OMPRuntime->emitTaskgraphTargetCall(
+ CGF, D, D.getBeginLoc(), OutlinedFnID, DeviceID,
+ Args.NumTeams.front(), Args.NumThreads.front(), KernelArgsPtr);
+ } else {
+ llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
+ cantFail(OMPRuntime->getOMPBuilder().emitKernelLaunch(
+ CGF.Builder, OutlinedFnID, EmitTargetCallFallbackCB, Args,
+ DeviceID, RTLoc, AllocaIP));
+ CGF.Builder.restoreIP(AfterIP);
+ }
};
- if (RequiresOuterTask)
- CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
- else
- OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
+ // The taskgraph (recording) path is never wrapped in a hidden helper task: a
+ // replayable target executes synchronously and __kmpc_taskgraph_target owns
+ // its own dependences. The ordinary (non-replayable) path keeps the helper
+ // task when depend/nowait require it.
+ //
+ // It still needs the inlined region, though: EmitBody evaluates the launch
+ // bounds (num_teams, thread_limit and, for a combined loop directive, the
+ // trip count), and those expressions can refer to variables captured by the
+ // target region, which only resolve while CapturedStmtInfo is installed.
+ auto &&EmitRecordingBody = [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ EmitBody(CGF, /*Recording=*/true);
+ };
+ auto &&EmitRecording = [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(),
+ EmitRecordingBody);
+ };
+ auto &&EmitOrdinaryBody = [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ EmitBody(CGF, /*Recording=*/false);
+ };
+ auto &&EmitOrdinary = [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ if (RequiresOuterTask)
+ CGF.EmitOMPTargetTaskBasedDirective(D, EmitOrdinaryBody, InputInfo);
+ else
+ OMPRuntime->emitInlinedDirective(CGF, D.getDirectiveKind(),
+ EmitOrdinaryBody);
+ };
+
+ // Record via the taskgraph entry point when the region is replayable (the
+ // runtime degrades to a synchronous normal launch if nothing is recording),
+ // and otherwise take the ordinary path. ReplayableCond already carries the
+ // enclosing taskgraph region's default (getOMPReplayableCond), which
+ // emitIfClause folds to just the recording path.
+ if (ReplayableCond) {
+ OMPRuntime->emitIfClause(CGF, ReplayableCond, EmitRecording, EmitOrdinary);
+ } else {
+ RegionCodeGenTy RCG(EmitOrdinary);
+ RCG(CGF);
+ }
}
static void
@@ -12279,7 +12440,8 @@ void CGOpenMPRuntime::emitTargetCall(
llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
const OMPLoopDirective &D)>
- SizeEmitter) {
+ SizeEmitter,
+ const Expr *ReplayableCond) {
if (!CGF.HaveInsertPoint())
return;
@@ -12288,6 +12450,10 @@ void CGOpenMPRuntime::emitTargetCall(
assert((OffloadingMandatory || OutlinedFn) && "Invalid outlined function!");
+ // This governs the ordinary (non-recording) path only. A recorded target
+ // region is handed to __kmpc_taskgraph_target (in
+ // emitTargetCallKernelLaunch) with its depend/nowait clauses forwarded, so
+ // that path is never wrapped in a hidden helper task and ignores this.
const bool RequiresOuterTask =
D.hasClausesOfKind<OMPDependClause>() ||
D.hasClausesOfKind<OMPNowaitClause>() ||
@@ -12307,16 +12473,16 @@ void CGOpenMPRuntime::emitTargetCall(
llvm::Value *MapTypesArray = nullptr;
llvm::Value *MapNamesArray = nullptr;
- auto &&TargetThenGen = [this, OutlinedFn, &D, &CapturedVars,
- RequiresOuterTask, &CS, OffloadingMandatory, Device,
- OutlinedFnID, &InputInfo, &MapTypesArray,
- &MapNamesArray, SizeEmitter](CodeGenFunction &CGF,
- PrePostActionTy &) {
- emitTargetCallKernelLaunch(this, OutlinedFn, D, CapturedVars,
- RequiresOuterTask, CS, OffloadingMandatory,
- Device, OutlinedFnID, InputInfo, MapTypesArray,
- MapNamesArray, SizeEmitter, CGF, CGM);
- };
+ auto &&TargetThenGen =
+ [this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
+ OffloadingMandatory, Device, OutlinedFnID, &InputInfo, &MapTypesArray,
+ &MapNamesArray, SizeEmitter,
+ ReplayableCond](CodeGenFunction &CGF, PrePostActionTy &) {
+ emitTargetCallKernelLaunch(
+ this, OutlinedFn, D, CapturedVars, RequiresOuterTask, CS,
+ OffloadingMandatory, Device, OutlinedFnID, InputInfo, MapTypesArray,
+ MapNamesArray, SizeEmitter, CGF, CGM, ReplayableCond);
+ };
auto &&TargetElseGen =
[this, OutlinedFn, &D, &CapturedVars, RequiresOuterTask, &CS,
@@ -12946,7 +13112,7 @@ void CGOpenMPRuntime::emitTargetDataCalls(
void CGOpenMPRuntime::emitTargetDataStandAloneCall(
CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
- const Expr *Device) {
+ const Expr *Device, const Expr *ReplayableCond) {
if (!CGF.HaveInsertPoint())
return;
@@ -12955,12 +13121,20 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
isa<OMPTargetUpdateDirective>(D)) &&
"Expecting either target enter, exit data, or update directives.");
+ // A depend/nowait clause normally needs a hidden helper task. The recording
+ // (taskgraph) path is the exception: it forwards the depend/nowait clauses
+ // to the __kmpc_taskgraph_target_*_data entry point, which satisfies them
+ // synchronously, so it is never wrapped in a helper task and ignores this.
+ // RequiresOuterTask therefore only governs the ordinary path below.
+ const bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() ||
+ D.hasClausesOfKind<OMPNowaitClause>();
+
CodeGenFunction::OMPTargetDataInfo InputInfo;
llvm::Value *MapTypesArray = nullptr;
llvm::Value *MapNamesArray = nullptr;
// Generate the code for the opening of the data environment.
- auto &&ThenGen = [this, &D, Device, &InputInfo, &MapTypesArray,
- &MapNamesArray](CodeGenFunction &CGF, PrePostActionTy &) {
+ auto &&EmitDataBody = [this, &D, Device, &InputInfo, &MapTypesArray,
+ &MapNamesArray](CodeGenFunction &CGF, bool Recording) {
// Emit device ID if any.
llvm::Value *DeviceID = nullptr;
if (Device) {
@@ -12970,123 +13144,142 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
}
- // Emit the number of elements in the offloading arrays.
- llvm::Constant *PointerNum =
- CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
-
- // Source location for the ident struct
- llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
+ if (Recording) {
+ // Record the directive via __kmpc_taskgraph_target_{enter,exit}_data /
+ // _update instead of issuing the data-mapper call directly. The taskgraph
+ // entry point satisfies the dependences itself; there is no enclosing
+ // helper task on this path.
+ emitTaskgraphTargetDataCall(
+ CGF, D, D.getBeginLoc(), DeviceID, InputInfo.NumberOfTargetItems,
+ InputInfo.BasePointersArray.emitRawPointer(CGF),
+ InputInfo.PointersArray.emitRawPointer(CGF),
+ InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray,
+ MapNamesArray, InputInfo.MappersArray.emitRawPointer(CGF));
+ return;
+ }
- SmallVector<llvm::Value *, 13> OffloadingArgs(
- {RTLoc, DeviceID, PointerNum,
- InputInfo.BasePointersArray.emitRawPointer(CGF),
- InputInfo.PointersArray.emitRawPointer(CGF),
- InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray, MapNamesArray,
- InputInfo.MappersArray.emitRawPointer(CGF)});
+ auto &&EmitNormalDataCall = [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ // Emit the number of elements in the offloading arrays.
+ llvm::Constant *PointerNum =
+ CGF.Builder.getInt32(InputInfo.NumberOfTargetItems);
+
+ // Source location for the ident struct
+ llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
+
+ SmallVector<llvm::Value *, 13> OffloadingArgs(
+ {RTLoc, DeviceID, PointerNum,
+ InputInfo.BasePointersArray.emitRawPointer(CGF),
+ InputInfo.PointersArray.emitRawPointer(CGF),
+ InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray,
+ MapNamesArray, InputInfo.MappersArray.emitRawPointer(CGF)});
+
+ // Select the right runtime function call for each standalone
+ // directive.
+ const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
+ RuntimeFunction RTLFn;
+ switch (D.getDirectiveKind()) {
+ case OMPD_target_enter_data:
+ RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
+ : OMPRTL___tgt_target_data_begin_mapper;
+ break;
+ case OMPD_target_exit_data:
+ RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
+ : OMPRTL___tgt_target_data_end_mapper;
+ break;
+ case OMPD_target_update:
+ RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
+ : OMPRTL___tgt_target_data_update_mapper;
+ break;
+ case OMPD_parallel:
+ case OMPD_for:
+ case OMPD_parallel_for:
+ case OMPD_parallel_master:
+ case OMPD_parallel_sections:
+ case OMPD_for_simd:
+ case OMPD_parallel_for_simd:
+ case OMPD_cancel:
+ case OMPD_cancellation_point:
+ case OMPD_ordered_standalone:
+ case OMPD_ordered_blockassoc:
+ case OMPD_threadprivate:
+ case OMPD_allocate:
+ case OMPD_task:
+ case OMPD_simd:
+ case OMPD_tile:
+ case OMPD_unroll:
+ case OMPD_sections:
+ case OMPD_section:
+ case OMPD_single:
+ case OMPD_master:
+ case OMPD_critical:
+ case OMPD_taskyield:
+ case OMPD_barrier:
+ case OMPD_taskwait:
+ case OMPD_taskgraph:
+ case OMPD_taskgroup:
+ case OMPD_atomic:
+ case OMPD_flush:
+ case OMPD_depobj:
+ case OMPD_scan:
+ case OMPD_teams:
+ case OMPD_target_data:
+ case OMPD_distribute:
+ case OMPD_distribute_simd:
+ case OMPD_distribute_parallel_for:
+ case OMPD_distribute_parallel_for_simd:
+ case OMPD_teams_distribute:
+ case OMPD_teams_distribute_simd:
+ case OMPD_teams_distribute_parallel_for:
+ case OMPD_teams_distribute_parallel_for_simd:
+ case OMPD_declare_simd:
+ case OMPD_declare_variant:
+ case OMPD_begin_declare_variant:
+ case OMPD_end_declare_variant:
+ case OMPD_declare_target:
+ case OMPD_end_declare_target:
+ case OMPD_declare_reduction:
+ case OMPD_declare_mapper:
+ case OMPD_taskloop:
+ case OMPD_taskloop_simd:
+ case OMPD_master_taskloop:
+ case OMPD_master_taskloop_simd:
+ case OMPD_parallel_master_taskloop:
+ case OMPD_parallel_master_taskloop_simd:
+ case OMPD_target:
+ case OMPD_target_simd:
+ case OMPD_target_teams_distribute:
+ case OMPD_target_teams_distribute_simd:
+ case OMPD_target_teams_distribute_parallel_for:
+ case OMPD_target_teams_distribute_parallel_for_simd:
+ case OMPD_target_teams:
+ case OMPD_target_parallel:
+ case OMPD_target_parallel_for:
+ case OMPD_target_parallel_for_simd:
+ case OMPD_requires:
+ case OMPD_metadirective:
+ case OMPD_unknown:
+ default:
+ llvm_unreachable("Unexpected standalone target data directive.");
+ break;
+ }
+ if (HasNowait) {
+ OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));
+ OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));
+ OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));
+ OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));
+ }
+ CGF.EmitRuntimeCall(
+ OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), RTLFn),
+ OffloadingArgs);
+ };
- // Select the right runtime function call for each standalone
- // directive.
- const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
- RuntimeFunction RTLFn;
- switch (D.getDirectiveKind()) {
- case OMPD_target_enter_data:
- RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper
- : OMPRTL___tgt_target_data_begin_mapper;
- break;
- case OMPD_target_exit_data:
- RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper
- : OMPRTL___tgt_target_data_end_mapper;
- break;
- case OMPD_target_update:
- RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper
- : OMPRTL___tgt_target_data_update_mapper;
- break;
- case OMPD_parallel:
- case OMPD_for:
- case OMPD_parallel_for:
- case OMPD_parallel_master:
- case OMPD_parallel_sections:
- case OMPD_for_simd:
- case OMPD_parallel_for_simd:
- case OMPD_cancel:
- case OMPD_cancellation_point:
- case OMPD_ordered_standalone:
- case OMPD_ordered_blockassoc:
- case OMPD_threadprivate:
- case OMPD_allocate:
- case OMPD_task:
- case OMPD_simd:
- case OMPD_tile:
- case OMPD_unroll:
- case OMPD_sections:
- case OMPD_section:
- case OMPD_single:
- case OMPD_master:
- case OMPD_critical:
- case OMPD_taskyield:
- case OMPD_barrier:
- case OMPD_taskwait:
- case OMPD_taskgraph:
- case OMPD_taskgroup:
- case OMPD_atomic:
- case OMPD_flush:
- case OMPD_depobj:
- case OMPD_scan:
- case OMPD_teams:
- case OMPD_target_data:
- case OMPD_distribute:
- case OMPD_distribute_simd:
- case OMPD_distribute_parallel_for:
- case OMPD_distribute_parallel_for_simd:
- case OMPD_teams_distribute:
- case OMPD_teams_distribute_simd:
- case OMPD_teams_distribute_parallel_for:
- case OMPD_teams_distribute_parallel_for_simd:
- case OMPD_declare_simd:
- case OMPD_declare_variant:
- case OMPD_begin_declare_variant:
- case OMPD_end_declare_variant:
- case OMPD_declare_target:
- case OMPD_end_declare_target:
- case OMPD_declare_reduction:
- case OMPD_declare_mapper:
- case OMPD_taskloop:
- case OMPD_taskloop_simd:
- case OMPD_master_taskloop:
- case OMPD_master_taskloop_simd:
- case OMPD_parallel_master_taskloop:
- case OMPD_parallel_master_taskloop_simd:
- case OMPD_target:
- case OMPD_target_simd:
- case OMPD_target_teams_distribute:
- case OMPD_target_teams_distribute_simd:
- case OMPD_target_teams_distribute_parallel_for:
- case OMPD_target_teams_distribute_parallel_for_simd:
- case OMPD_target_teams:
- case OMPD_target_parallel:
- case OMPD_target_parallel_for:
- case OMPD_target_parallel_for_simd:
- case OMPD_requires:
- case OMPD_metadirective:
- case OMPD_unknown:
- default:
- llvm_unreachable("Unexpected standalone target data directive.");
- break;
- }
- if (HasNowait) {
- OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));
- OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));
- OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.Int32Ty));
- OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.VoidPtrTy));
- }
- CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(), RTLFn),
- OffloadingArgs);
+ RegionCodeGenTy RCG(EmitNormalDataCall);
+ RCG(CGF);
};
- auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray,
- &MapNamesArray](CodeGenFunction &CGF,
- PrePostActionTy &) {
+ auto &&TargetThenGen = [this, &EmitDataBody, &D, &InputInfo, &MapTypesArray,
+ &MapNamesArray, RequiresOuterTask, ReplayableCond](
+ CodeGenFunction &CGF, PrePostActionTy &) {
// Fill up the arrays with all the mapped variables.
MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
CGOpenMPRuntime::TargetDataInfo Info;
@@ -13095,9 +13288,6 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
/*IsNonContiguous=*/true, /*ForEndCall=*/false);
- bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>() ||
- D.hasClausesOfKind<OMPNowaitClause>();
-
InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;
InputInfo.BasePointersArray = Address(Info.RTArgs.BasePointersArray,
CGF.VoidPtrTy, CGM.getPointerAlign());
@@ -13109,10 +13299,37 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
Address(Info.RTArgs.MappersArray, CGF.VoidPtrTy, CGM.getPointerAlign());
MapTypesArray = Info.RTArgs.MapTypesArray;
MapNamesArray = Info.RTArgs.MapNamesArray;
- if (RequiresOuterTask)
- CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo);
- else
- emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen);
+
+ // The taskgraph (recording) path is never wrapped in a hidden helper task:
+ // a replayable target-data directive executes synchronously and the
+ // taskgraph entry point owns its own dependences. The ordinary
+ // (non-replayable) path keeps the helper task when depend/nowait require
+ // it.
+ auto &&EmitRecording = [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ EmitDataBody(CGF, /*Recording=*/true);
+ };
+ auto &&EmitOrdinaryBody = [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ EmitDataBody(CGF, /*Recording=*/false);
+ };
+ auto &&EmitOrdinary = [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ if (RequiresOuterTask)
+ CGF.EmitOMPTargetTaskBasedDirective(D, EmitOrdinaryBody, InputInfo);
+ else
+ emitInlinedDirective(CGF, D.getDirectiveKind(), EmitOrdinaryBody);
+ };
+
+ // Record via the taskgraph entry point when the directive is replayable
+ // (the runtime degrades to a synchronous normal mapper call if nothing is
+ // recording), and otherwise issue the ordinary mapper call.
+ // ReplayableCond already carries the enclosing taskgraph region's default
+ // (getOMPReplayableCond), which emitIfClause folds to just the recording
+ // path.
+ if (ReplayableCond) {
+ emitIfClause(CGF, ReplayableCond, EmitRecording, EmitOrdinary);
+ } else {
+ RegionCodeGenTy RCG(EmitOrdinary);
+ RCG(CGF);
+ }
};
if (IfCond) {
@@ -14582,7 +14799,8 @@ void CGOpenMPSIMDRuntime::emitTargetCall(
llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
const OMPLoopDirective &D)>
- SizeEmitter) {
+ SizeEmitter,
+ const Expr *ReplayableCond) {
llvm_unreachable("Not supported in SIMD-only mode");
}
@@ -14622,7 +14840,7 @@ void CGOpenMPSIMDRuntime::emitTargetDataCalls(
void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall(
CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
- const Expr *Device) {
+ const Expr *Device, const Expr *ReplayableCond) {
llvm_unreachable("Not supported in SIMD-only mode");
}
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 7e1f9bdfc687d1..5e9f639506be1f 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -136,6 +136,10 @@ struct OMPTaskDataTy final {
bool BodyConstructsPrivates = false;
};
+/// Gather the depend clauses of \p S into \p Data.Dependences, collapsing
+/// 'omp_all_memory' as the runtime expects.
+void buildDependences(const OMPExecutableDirective &S, OMPTaskDataTy &Data);
+
/// Class intended to support codegen of all kind of the reduction clauses.
class ReductionCodeGen {
private:
@@ -1472,13 +1476,44 @@ class CGOpenMPRuntime {
/// target directive, or null if no device clause is used and device modifier.
/// \param SizeEmitter Callback to emit number of iterations for loop-based
/// directives.
+ /// \param ReplayableCond Condition of the replayable clause associated with
+ /// the target directive (a constant-true literal if the clause has no
+ /// argument), or null if no replayable clause is used.
virtual void emitTargetCall(
CodeGenFunction &CGF, const OMPExecutableDirective &D,
llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
const OMPLoopDirective &D)>
- SizeEmitter);
+ SizeEmitter,
+ const Expr *ReplayableCond = nullptr);
+
+ /// Record a target region nested inside a taskgraph by emitting a
+ /// __kmpc_taskgraph_target call. \p KernelArgsPtr is a pointer to a populated
+ /// __tgt_kernel_arguments struct; \p NumTeams / \p NumThreads / \p DeviceID
+ /// are the launch parameters and \p OutlinedFnID is the host pointer used to
+ /// identify the region. Depend clauses of \p D are forwarded to the entry
+ /// point, which satisfies them synchronously; a replayable target is never
+ /// wrapped in a hidden helper task.
+ void emitTaskgraphTargetCall(CodeGenFunction &CGF,
+ const OMPExecutableDirective &D,
+ SourceLocation Loc, llvm::Value *OutlinedFnID,
+ llvm::Value *DeviceID, llvm::Value *NumTeams,
+ llvm::Value *NumThreads,
+ llvm::Value *KernelArgsPtr);
+
+ /// Record a standalone 'target {enter|exit} data' / 'target update' directive
+ /// nested inside a taskgraph by emitting the matching
+ /// __kmpc_taskgraph_target_{enter,exit}_data / _update call. The map-info
+ /// arrays are passed directly, and depend clauses of \p D are forwarded to
+ /// the entry point, which satisfies them synchronously; a replayable
+ /// target-data directive is never wrapped in a hidden helper task.
+ void emitTaskgraphTargetDataCall(
+ CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc,
+ llvm::Value *DeviceID, unsigned NumTargetItems,
+ llvm::Value *BasePointersArray, llvm::Value *PointersArray,
+ llvm::Value *SizesArray, llvm::Value *MapTypesArray,
+ llvm::Value *MapNamesArray, llvm::Value *MappersArray);
/// Emit the target regions enclosed in \a GD function definition or
/// the function itself in case it is a valid device function. Returns true if
@@ -1569,10 +1604,12 @@ class CGOpenMPRuntime {
/// directive, or null if no if clause is used.
/// \param Device Expression evaluated in device clause associated with the
/// target directive, or null if no device clause is used.
- virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
- const OMPExecutableDirective &D,
- const Expr *IfCond,
- const Expr *Device);
+ /// \param ReplayableCond Condition of the replayable clause associated with
+ /// the target directive (a constant-true literal if the clause has no
+ /// argument), or null if no replayable clause is used.
+ virtual void emitTargetDataStandAloneCall(
+ CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
+ const Expr *Device, const Expr *ReplayableCond = nullptr);
/// Marks function \a Fn with properly mangled versions of vector functions.
/// \param FD Function marked as 'declare simd'.
@@ -2315,7 +2352,8 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier> Device,
llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
const OMPLoopDirective &D)>
- SizeEmitter) override;
+ SizeEmitter,
+ const Expr *ReplayableCond = nullptr) override;
/// Emit the target regions enclosed in \a GD function definition or
/// the function itself in case it is a valid device function. Returns true if
@@ -2373,10 +2411,9 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
/// directive, or null if no if clause is used.
/// \param Device Expression evaluated in device clause associated with the
/// target directive, or null if no device clause is used.
- void emitTargetDataStandAloneCall(CodeGenFunction &CGF,
- const OMPExecutableDirective &D,
- const Expr *IfCond,
- const Expr *Device) override;
+ void emitTargetDataStandAloneCall(
+ CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
+ const Expr *Device, const Expr *ReplayableCond = nullptr) override;
/// Emit initialization for doacross loop nesting support.
/// \param D Loop-based construct used in doacross nesting construct.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 3babe21a23dac5..504abdba73666b 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -5443,8 +5443,8 @@ class CheckVarsEscapingUntiedTaskDeclContext final
};
} // anonymous namespace
-static void buildDependences(const OMPExecutableDirective &S,
- OMPTaskDataTy &Data) {
+void clang::CodeGen::buildDependences(const OMPExecutableDirective &S,
+ OMPTaskDataTy &Data) {
// First look for 'omp_all_memory' and add this first.
bool OmpAllMemory = false;
@@ -6153,19 +6153,14 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
IntegerLiteral IfCond(getContext(), TrueOrFalse,
getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
SourceLocation());
- const Expr *ReplayableCond = nullptr;
- if (auto *RC = S.getSingleClause<OMPReplayableClause>()) {
- ReplayableCond = RC->getCondition();
- if (!ReplayableCond) {
- ReplayableCond = IntegerLiteral::Create(
- getContext(), llvm::APInt(32, 1),
- getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
- SourceLocation());
- }
- }
+ // The hidden helper task this emits is never itself a taskgraph node: a
+ // recorded target region is handed to __kmpc_taskgraph_target, which owns
+ // the dependences and needs no helper task, so every caller of this reaches
+ // it on the ordinary (non-recording) path. Passing the directive's own
+ // replayable condition down here would record the helper instead.
CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
SharedsTy, CapturedStruct, &IfCond,
- ReplayableCond, Data);
+ /*ReplayableCond=*/nullptr, Data);
}
void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
@@ -7773,7 +7768,8 @@ static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
return nullptr;
};
CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
- SizeEmitter);
+ SizeEmitter,
+ getOMPReplayableCond(CGF, S));
}
static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
@@ -8574,7 +8570,8 @@ void CodeGenFunction::EmitOMPTargetEnterDataDirective(
Device = C->getDevice();
OMPLexicalScope Scope(*this, S, OMPD_task);
- CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
+ CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(
+ *this, S, IfCond, Device, getOMPReplayableCond(*this, S));
}
void CodeGenFunction::EmitOMPTargetExitDataDirective(
@@ -8595,7 +8592,8 @@ void CodeGenFunction::EmitOMPTargetExitDataDirective(
Device = C->getDevice();
OMPLexicalScope Scope(*this, S, OMPD_task);
- CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
+ CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(
+ *this, S, IfCond, Device, getOMPReplayableCond(*this, S));
}
static void emitTargetParallelRegion(CodeGenFunction &CGF,
@@ -9059,7 +9057,8 @@ void CodeGenFunction::EmitOMPTargetUpdateDirective(
Device = C->getDevice();
OMPLexicalScope Scope(*this, S, OMPD_task);
- CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
+ CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(
+ *this, S, IfCond, Device, getOMPReplayableCond(*this, S));
}
void CodeGenFunction::EmitOMPGenericLoopDirective(
diff --git a/clang/test/OpenMP/taskgraph_target_codegen.cpp b/clang/test/OpenMP/taskgraph_target_codegen.cpp
new file mode 100644
index 00000000000000..0e9281909ac20e
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_target_codegen.cpp
@@ -0,0 +1,53 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-linux-gnu -fopenmp-targets=x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// Verifies that an 'omp target' region lexically nested inside an
+// 'omp taskgraph' is recorded via __kmpc_taskgraph_target, while the same
+// region outside a taskgraph still lowers to the ordinary __tgt_target_kernel
+// launch. The taskgraph entry point receives the populated
+// __tgt_kernel_arguments struct (last pointer argument) together with the
+// launch parameters and the depend list.
+
+void outside() {
+ int x = 0;
+#pragma omp target map(tofrom : x)
+ { x++; }
+}
+
+void inside() {
+ int x = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : x)
+ { x++; }
+ }
+}
+
+void inside_depend() {
+ int x = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : x) depend(out : x) nowait
+ { x++; }
+ }
+}
+
+// Outside a taskgraph: ordinary kernel launch, no taskgraph recording.
+// CHECK-LABEL: define {{.*}}@_Z7outsidev
+// CHECK: call i32 @__tgt_target_kernel(
+// CHECK-NOT: @__kmpc_taskgraph_target
+
+// Inside a taskgraph: recorded via __kmpc_taskgraph_target. The trailing
+// arguments are the __tgt_kernel_arguments struct and the (currently null)
+// relocation callback; ndeps (arg 3) is 0 and the dep_list (arg 4) is null for
+// a target with no depend clause.
+// CHECK: call i32 @__kmpc_taskgraph_target(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr %kernel_args{{[0-9.]*}}, ptr null)
+
+// Inside a taskgraph with depend(out:) + nowait: ndeps is non-zero, the
+// dep_list points at the emitted kmp_depend_info array, and has_no_wait is 1.
+// CHECK: call i32 @__kmpc_taskgraph_target(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 1, ptr {{[^,]+}}, i32 1, i64 {{[^,]+}}, i32 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr %kernel_args{{[0-9.]*}}, ptr null)
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_target_data_codegen.cpp b/clang/test/OpenMP/taskgraph_target_data_codegen.cpp
new file mode 100644
index 00000000000000..05f8e4ef8cd37a
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_target_data_codegen.cpp
@@ -0,0 +1,44 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-linux-gnu -fopenmp-targets=x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// Verifies that 'omp target enter data', 'omp target exit data', and
+// 'omp target update' directives lexically nested inside an 'omp taskgraph'
+// are recorded via the matching __kmpc_taskgraph_target_{enter,exit}_data /
+// _update entry points (with the map-info arrays forwarded), while the same
+// directives outside a taskgraph still lower to the ordinary
+// __tgt_target_data_*_mapper calls.
+
+void outside() {
+ int x = 0;
+#pragma omp target enter data map(to : x)
+#pragma omp target exit data map(from : x)
+}
+
+void inside() {
+ int x = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target enter data map(to : x)
+#pragma omp target update to(x)
+#pragma omp target exit data map(from : x)
+ }
+}
+
+// Outside a taskgraph: ordinary data mapper calls, no taskgraph recording.
+// CHECK-LABEL: define {{.*}}@_Z7outsidev
+// CHECK: call void @__tgt_target_data_begin_mapper(
+// CHECK: call void @__tgt_target_data_end_mapper(
+// CHECK-NOT: @__kmpc_taskgraph_target_
+
+// Inside a taskgraph: recorded via the matching taskgraph entry points. With
+// no depend clause, ndeps (arg 3) is 0 and dep_list (arg 4) is null; the seven
+// map-info pointers are forwarded, followed by the (currently null) relocation
+// callback as the trailing argument.
+// CHECK: call void @__kmpc_taskgraph_target_enter_data(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr null)
+// CHECK: call void @__kmpc_taskgraph_target_update(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr null)
+// CHECK: call void @__kmpc_taskgraph_target_exit_data(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr null)
+
+#endif
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index a1ed0d2b065759..ddb990cf3e1b54 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -2617,6 +2617,19 @@ class OpenMPIRBuilder {
Value *NumThreads, Value *HostPtr,
ArrayRef<Value *> KernelArgs);
+ /// Allocate and populate a \c __tgt_kernel_arguments struct from \p
+ /// KernelArgs, returning a pointer to it. This is the storage that
+ /// emitTargetKernel passes to \c __tgt_target_kernel; it is exposed so that
+ /// alternative callers (e.g. taskgraph recording) can pass the same struct to
+ /// a different runtime entry point.
+ ///
+ /// \param Loc The location at which the request originated and is fulfilled.
+ /// \param AllocaIP The insertion point to be used for alloca instructions.
+ /// \param KernelArgs Array of arguments to the kernel.
+ LLVM_ABI Value *emitKernelArgsStruct(const LocationDescription &Loc,
+ InsertPointTy AllocaIP,
+ ArrayRef<Value *> KernelArgs);
+
/// Generate a flush runtime call.
///
/// \param Loc The location at which the request originated and is fulfilled.
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 1d41c41aad3e68..96bc290450c4c5 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -377,6 +377,24 @@ __OMP_RTL(__kmpc_taskgraph_taskwait, false, Void, IdentPtr, Int32, Int32,
VoidPtr, Int32)
__OMP_RTL(__kmpc_taskgraph_taskred_init, false, /* kmp_taskgroup */ VoidPtr,
Int32, Int32, VoidPtr)
+__OMP_RTL(__kmpc_taskgraph_target, false, Int32, IdentPtr, Int32, Int32,
+ /* kmp_depend_info_t */ VoidPtr, Int32, Int64, Int32, Int32, VoidPtr,
+ /* __tgt_kernel_arguments */ VoidPtr,
+ /* kmp_target_kernel_relocate_t */ VoidPtr)
+__OMP_RTL(__kmpc_taskgraph_target_enter_data, false, Void, IdentPtr, Int32,
+ Int32, /* kmp_depend_info_t */ VoidPtr, Int32, Int64, Int32,
+ VoidPtrPtr, VoidPtrPtr, Int64Ptr, Int64Ptr,
+ /* map_var_info_t */ VoidPtrPtr, VoidPtrPtr,
+ /* kmp_target_data_relocate_t */ VoidPtr)
+__OMP_RTL(__kmpc_taskgraph_target_exit_data, false, Void, IdentPtr, Int32,
+ Int32, /* kmp_depend_info_t */ VoidPtr, Int32, Int64, Int32,
+ VoidPtrPtr, VoidPtrPtr, Int64Ptr, Int64Ptr,
+ /* map_var_info_t */ VoidPtrPtr, VoidPtrPtr,
+ /* kmp_target_data_relocate_t */ VoidPtr)
+__OMP_RTL(__kmpc_taskgraph_target_update, false, Void, IdentPtr, Int32, Int32,
+ /* kmp_depend_info_t */ VoidPtr, Int32, Int64, Int32, VoidPtrPtr,
+ VoidPtrPtr, Int64Ptr, Int64Ptr, /* map_var_info_t */ VoidPtrPtr,
+ VoidPtrPtr, /* kmp_target_data_relocate_t */ VoidPtr)
__OMP_RTL(__kmpc_omp_task_begin_if0, false, Void, IdentPtr, Int32,
/* kmp_task_t */ VoidPtr)
__OMP_RTL(__kmpc_omp_task_complete_if0, false, Void, IdentPtr, Int32,
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 2fbbeefff2d5c6..2da95501750b2c 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -1544,13 +1544,9 @@ OpenMPIRBuilder::createCancellationPoint(const LocationDescription &Loc,
return Builder.saveIP();
}
-OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
- const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return,
- Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads,
- Value *HostPtr, ArrayRef<Value *> KernelArgs) {
- if (!updateToLocation(Loc))
- return Loc.IP;
-
+Value *OpenMPIRBuilder::emitKernelArgsStruct(const LocationDescription &Loc,
+ InsertPointTy AllocaIP,
+ ArrayRef<Value *> KernelArgs) {
Builder.restoreIP(AllocaIP);
auto *KernelArgsPtr =
Builder.CreateAlloca(OpenMPIRBuilder::KernelArgs, nullptr, "kernel_args");
@@ -1564,6 +1560,18 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
M.getDataLayout().getPrefTypeAlign(KernelArgs[I]->getType()));
}
+ return KernelArgsPtr;
+}
+
+OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel(
+ const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return,
+ Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads,
+ Value *HostPtr, ArrayRef<Value *> KernelArgs) {
+ if (!updateToLocation(Loc))
+ return Loc.IP;
+
+ Value *KernelArgsPtr = emitKernelArgsStruct(Loc, AllocaIP, KernelArgs);
+
SmallVector<Value *> OffloadingArgs{Ident, DeviceID, NumTeams,
NumThreads, HostPtr, KernelArgsPtr};
diff --git a/offload/test/offloading/taskgraph_target.c b/offload/test/offloading/taskgraph_target.c
new file mode 100644
index 00000000000000..8c83dead471068
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target.c
@@ -0,0 +1,41 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A 'target' region lexically nested in an 'omp taskgraph', encountered once,
+// so the graph is only recorded: __kmpc_taskgraph_target records the region
+// and launches it through __tgt_target_kernel, producing the same result as an
+// ordinary launch.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 1024;
+ int *a = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = i;
+
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : a[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] * 2 + 1;
+ }
+ }
+
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (a[i] != 2 * i + 1)
+ ++errors;
+
+ // CHECK: taskgraph target errors: 0
+ printf("taskgraph target errors: %d\n", errors);
+ free(a);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_data.c b/offload/test/offloading/taskgraph_target_data.c
new file mode 100644
index 00000000000000..f45a97508467e6
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_data.c
@@ -0,0 +1,59 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Target-data constructs nested in an 'omp taskgraph', encountered once, so
+// the graph is only recorded: 'target enter data', 'target update' (to/from)
+// and 'target exit data' are recorded by
+// __kmpc_taskgraph_target_{enter,exit}_data and __kmpc_taskgraph_target_update
+// and issued through the libomptarget *_mapper calls.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 1024;
+ int *a = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = i;
+
+#pragma omp taskgraph
+ {
+#pragma omp target enter data map(to : a[0 : N])
+
+#pragma omp target
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] + 5;
+ }
+
+ // Pull values back to the host, modify, push them back.
+#pragma omp target update from(a[0 : N])
+ for (int i = 0; i < N; ++i)
+ a[i] *= 2;
+#pragma omp target update to(a[0 : N])
+
+#pragma omp target
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] - 1;
+ }
+
+#pragma omp target exit data map(from : a[0 : N])
+ }
+
+ // Expected: ((i + 5) * 2) - 1 = 2 * i + 9
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (a[i] != 2 * i + 9)
+ ++errors;
+
+ // CHECK: taskgraph target data errors: 0
+ printf("taskgraph target data errors: %d\n", errors);
+ free(a);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_irreducible.c b/offload/test/offloading/taskgraph_target_irreducible.c
new file mode 100644
index 00000000000000..82c079c352877f
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_irreducible.c
@@ -0,0 +1,81 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// An NxN lattice "wavefront" of 'target' kernels inside an 'omp taskgraph',
+// re-encountered several times. Cell (i,j) depends on (i-1,j) and (i,j-1) --
+// the canonical non-series-parallel (irreducible) dependence graph. The region
+// builder carves it into a single irreducible region and streams its explicit
+// intra-kernel edges to libomptarget; on replay a plugin that claimed the
+// subgraph dispatches the kernels honoring those edges, and otherwise libomp
+// issues them itself, honoring the same edges. Either way the wavefront
+// values (binomial C(i+j, i)) must come out right every iteration.
+//
+// `nowait` is what keeps the wavefront edges the only ones. A recorded target
+// without it generates an undeferred task, so encounter order is a dependence
+// too, and the extra edges make the graph a total order -- which is trivially
+// series-parallel, so the carve would come out sequential and the irreducible
+// path this test is named for would never run.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 4
+#define ITERS 4
+
+int main() {
+ long long *v = (long long *)malloc(N * N * sizeof(long long));
+ long long ref[N][N];
+ for (int i = 0; i < N; ++i)
+ for (int j = 0; j < N; ++j)
+ ref[i][j] =
+ (i == 0 && j == 0)
+ ? 1
+ : ((i > 0 ? ref[i - 1][j] : 0) + (j > 0 ? ref[i][j - 1] : 0));
+
+ int errors = 0;
+ for (int it = 0; it < ITERS; ++it) {
+ for (int k = 0; k < N * N; ++k)
+ v[k] = -1;
+#pragma omp taskgraph
+ {
+ for (int i = 0; i < N; ++i)
+ for (int j = 0; j < N; ++j) {
+ int me = i * N + j;
+ int up = (i - 1) * N + j;
+ int left = i * N + (j - 1);
+ // clang-format off
+ if (i == 0 && j == 0) {
+#pragma omp target nowait map(tofrom : v[0 : N * N]) depend(out : v[me])
+ { v[me] = 1; }
+ } else if (i == 0) {
+#pragma omp target nowait map(tofrom : v[0 : N * N]) \
+ depend(in : v[left]) depend(out : v[me])
+ { v[me] = v[left]; }
+ } else if (j == 0) {
+#pragma omp target nowait map(tofrom : v[0 : N * N]) \
+ depend(in : v[up]) depend(out : v[me])
+ { v[me] = v[up]; }
+ } else {
+#pragma omp target nowait map(tofrom : v[0 : N * N]) \
+ depend(in : v[up], v[left]) depend(out : v[me])
+ { v[me] = v[up] + v[left]; }
+ }
+ // clang-format on
+ }
+ }
+ for (int i = 0; i < N; ++i)
+ for (int j = 0; j < N; ++j)
+ if (v[i * N + j] != ref[i][j])
+ ++errors;
+ }
+
+ // CHECK: taskgraph irreducible target errors: 0
+ printf("taskgraph irreducible target errors: %d\n", errors);
+ free(v);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_nowait_schedule.c b/offload/test/offloading/taskgraph_target_nowait_schedule.c
new file mode 100644
index 00000000000000..055f8ad351e6fb
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_nowait_schedule.c
@@ -0,0 +1,81 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env KMP_TASKGRAPH_TRACE=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A recorded target construct without `nowait` generates an undeferred
+// (included) task, so encounter order is a dependence and the recorded schedule
+// has to be a chain. With `nowait` the task is deferrable, program order says
+// nothing, and independent targets stay in one parallel region -- which is the
+// only way to get a parallel recorded graph out of a sequence of targets, so it
+// is worth pinning down that the encounter-order chaining does not swallow it.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 256;
+ int *a = (int *)malloc(N * sizeof(int));
+ int *b = (int *)malloc(N * sizeof(int));
+ int *c = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = b[i] = c[i] = i;
+
+ for (int it = 0; it < 2; ++it) {
+ // No nowait: three undeferred tasks, so a chain.
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : a[0 : N])
+ for (int i = 0; i < N; ++i)
+ a[i] += 1;
+#pragma omp target map(tofrom : b[0 : N])
+ for (int i = 0; i < N; ++i)
+ b[i] += 1;
+#pragma omp target map(tofrom : c[0 : N])
+ for (int i = 0; i < N; ++i)
+ c[i] += 1;
+ }
+
+ // nowait, disjoint data, no depend clauses: nothing orders these.
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : a[0 : N]) nowait
+ for (int i = 0; i < N; ++i)
+ a[i] += 1;
+#pragma omp target map(tofrom : b[0 : N]) nowait
+ for (int i = 0; i < N; ++i)
+ b[i] += 1;
+#pragma omp target map(tofrom : c[0 : N]) nowait
+ for (int i = 0; i < N; ++i)
+ c[i] += 1;
+ }
+ }
+
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (a[i] != i + 4 || b[i] != i + 4 || c[i] != i + 4)
+ ++errors;
+
+ // CHECK: Processed taskgraph
+ // CHECK-NEXT: sequential {
+ // CHECK-NEXT: target: 0x{{[[:xdigit:]]+}}
+ // CHECK-NEXT: target: 0x{{[[:xdigit:]]+}}
+ // CHECK-NEXT: target: 0x{{[[:xdigit:]]+}}
+ // CHECK-NEXT: }
+
+ // CHECK: Processed taskgraph
+ // CHECK-NEXT: parallel {
+ // CHECK-NEXT: target: 0x{{[[:xdigit:]]+}}
+ // CHECK-NEXT: target: 0x{{[[:xdigit:]]+}}
+ // CHECK-NEXT: target: 0x{{[[:xdigit:]]+}}
+ // CHECK-NEXT: }
+
+ // CHECK: taskgraph target nowait schedule errors: 0
+ printf("taskgraph target nowait schedule errors: %d\n", errors);
+ free(c);
+ free(b);
+ free(a);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_record.c b/offload/test/offloading/taskgraph_target_record.c
new file mode 100644
index 00000000000000..a03aceee71cc64
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_record.c
@@ -0,0 +1,63 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Exercises *recording + replay* of target / target-data constructs in a
+// taskgraph: a taskgraph body containing 'target enter data', a 'target'
+// kernel, a 'target update', and 'target exit data' (chained with depend
+// clauses) is captured as four discriminated TARGET regions that participate
+// in the clone ring, then re-entered several times.
+//
+// The first (recording) pass forwards each construct to libomptarget live; the
+// processed graph is then handed to libomptarget, and subsequent passes are
+// replayed through it (__tgt_taskgraph_replay), re-issuing every recorded data
+// op and kernel. Each pass therefore applies the device transform once, so
+// the result accumulates across passes. This validates that recording, graph
+// build, replay (re-issue + clone-ring linkage), and teardown all behave.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 1024;
+ const int Reps = 3;
+ int *a = (int *)malloc(N * sizeof(int));
+ int *ref = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = ref[i] = i;
+
+ for (int rep = 0; rep < Reps; ++rep) {
+#pragma omp taskgraph
+ {
+#pragma omp target enter data map(to : a[0 : N]) depend(inout : a[0])
+#pragma omp target map(alloc : a[0 : N]) depend(inout : a[0])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] * 2 + 1;
+ }
+#pragma omp target update from(a[0 : N]) depend(inout : a[0])
+#pragma omp target exit data map(release : a[0 : N]) depend(inout : a[0])
+ }
+ }
+
+ // The device transform a -> 2a+1 is applied once per pass (recording +
+ // replays), so the result matches the same transform applied Reps times.
+ for (int rep = 0; rep < Reps; ++rep)
+ for (int i = 0; i < N; ++i)
+ ref[i] = ref[i] * 2 + 1;
+
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (a[i] != ref[i])
+ ++errors;
+
+ // CHECK: taskgraph target record errors: 0
+ printf("taskgraph target record errors: %d\n", errors);
+ free(a);
+ free(ref);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_replay.c b/offload/test/offloading/taskgraph_target_replay.c
new file mode 100644
index 00000000000000..4567f37a1f6c7b
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_replay.c
@@ -0,0 +1,47 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Re-encounters an 'omp taskgraph' that contains a 'target' region several
+// times. The first encounter records the graph and launches live; because the
+// graph contains a target, the processed region tree is handed to libomptarget
+// (__tgt_taskgraph_start/.../_finalize) and every subsequent encounter is
+// replayed through libomptarget (__tgt_taskgraph_replay) instead of the host
+// exec_descr path. All encounters must produce the same per-iteration effect.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 1024;
+ const int Iters = 4;
+ int *a = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = i;
+
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : a[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] + 1;
+ }
+ }
+ }
+
+ // Each of the Iters encounters adds 1, so a[i] == i + Iters.
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (a[i] != i + Iters)
+ ++errors;
+
+ // CHECK: taskgraph target replay errors: 0
+ printf("taskgraph target replay errors: %d\n", errors);
+ free(a);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_replay_mixed.c b/offload/test/offloading/taskgraph_target_replay_mixed.c
new file mode 100644
index 00000000000000..668dc2daa3b7c0
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_replay_mixed.c
@@ -0,0 +1,57 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A mixed host + target 'omp taskgraph', re-encountered several times. A host
+// 'task' (depend out) runs before a 'target' kernel (depend in) each encounter.
+// On replay libomp runs the recorded host task and re-issues the target kernel
+// through the standard libomptarget launch path.
+// The dependency ordering (host then device) must be preserved every time.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 1024;
+ const int Iters = 4;
+ int *a = (int *)malloc(N * sizeof(int));
+ int *ref = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = ref[i] = i;
+
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp task depend(out : a[0 : N]) shared(a)
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] + 1; // host side, runs first
+ }
+#pragma omp target map(tofrom : a[0 : N]) depend(in : a[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] * 2; // device side, runs second
+ }
+ }
+ }
+
+ // Host reference: same (x+1)*2 transform applied Iters times.
+ for (int it = 0; it < Iters; ++it)
+ for (int i = 0; i < N; ++i)
+ ref[i] = (ref[i] + 1) * 2;
+
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (a[i] != ref[i])
+ ++errors;
+
+ // CHECK: taskgraph mixed replay errors: 0
+ printf("taskgraph mixed replay errors: %d\n", errors);
+ free(a);
+ free(ref);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_replayable.c b/offload/test/offloading/taskgraph_target_replayable.c
new file mode 100644
index 00000000000000..4833a259c26633
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_replayable.c
@@ -0,0 +1,97 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Exercises the 'replayable' clause on target / target-data constructs that are
+// NOT lexically nested inside an 'omp taskgraph'. The compiler routes these to
+// the __kmpc_taskgraph_target* entry points; since no taskgraph is being
+// recorded here, those must take the ordinary libomptarget path and produce
+// the same result as a plain target. They satisfy dependences themselves,
+// and a target region runs synchronously, its nowait dropped.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 1024;
+ int *a = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = i;
+
+ // replayable with no argument (constant-true): single taskgraph entry-point
+ // call that degrades to a normal launch since nothing is recording.
+#pragma omp target map(tofrom : a[0 : N]) replayable
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] * 2 + 1;
+ }
+
+ // replayable(expr) with a true condition: takes the taskgraph entry-point
+ // branch, which degrades to a normal launch.
+ int t = 1;
+#pragma omp target map(tofrom : a[0 : N]) replayable(t)
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] + 3;
+ }
+
+ // replayable(expr) with a false condition: takes the ordinary launch branch.
+ int f = 0;
+#pragma omp target map(tofrom : a[0 : N]) replayable(f)
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] - 2;
+ }
+
+ // replayable target with depend + nowait outside a taskgraph: a replayable
+ // target executes synchronously, so no hidden helper task is created. The
+ // dependences are forwarded to __kmpc_taskgraph_target (which satisfies them)
+ // and the nowait is dropped, so the following taskwait is a no-op.
+#pragma omp target map(tofrom : a[0 : N]) depend(inout : a[0]) nowait replayable
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] + 10;
+ }
+#pragma omp taskwait
+
+ // replayable target-data (enter / update / exit) outside a taskgraph.
+#pragma omp target enter data map(to : a[0 : N]) replayable
+#pragma omp target map(alloc : a[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] * 3;
+ }
+#pragma omp target update from(a[0 : N]) replayable(t)
+ for (int i = 0; i < N; ++i)
+ a[i] += 1;
+#pragma omp target update to(a[0 : N]) replayable
+#pragma omp target map(alloc : a[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] - 4;
+ }
+#pragma omp target exit data map(from : a[0 : N]) replayable
+
+ // Trace per element:
+ // i
+ // *2+1 -> 2i+1
+ // +3 -> 2i+4
+ // -2 -> 2i+2
+ // +10 -> 2i+12
+ // *3 -> 6i+36
+ // +1 -> 6i+37
+ // -4 -> 6i+33
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (a[i] != 6 * i + 33)
+ ++errors;
+
+ // CHECK: replayable target errors: 0
+ printf("replayable target errors: %d\n", errors);
+ free(a);
+ return errors != 0;
+}
>From 88c78cfa8483e7ac7d2385ffe6b46f261f8086b2 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 19 Aug 2026 10:47:33 -0500
Subject: [PATCH 16/29] [Clang][Offload] Implement 'saved' modifier for
firstprivate aggregates
This patch implements front-end and runtime parts for the 'saved'
modifier for firstprivate aggregates. Scalars or pointers (word-size
data) are already handled implicitly because they are passed by value in
mapping tables. For aggregates though, the actual data may be in the
stack frame of the recording task: for that reason, we must snapshot
the data and redirect the appropriate pointers into that snapshot.
This is (part of) the "saved data environment" described in the spec
(OpenMP 6.0, "14.3 taskgraph Construct"), though it's distributed in
this implementation, rather than all being in one place in some given
taskgraph record.
Variables declared within the taskgraph region, but not within a task,
that are listed in firstprivate clauses of a construct within the
taskgraph are also treated as "saved", as per the spec.
Tests have been added to cover new functionality.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/225721
---
clang/lib/CodeGen/CGOpenMPRuntime.cpp | 60 ++++++++++++++----
.../OpenMP/taskgraph_target_saved_codegen.c | 54 ++++++++++++++++
.../llvm/Frontend/OpenMP/OMPConstants.h | 3 +
offload/include/omptarget.h | 25 ++++++++
offload/libomptarget/interface.cpp | 25 ++++++++
.../taskgraph_firstprivate_saved_aggregate.c | 61 +++++++++++++++++++
6 files changed, 215 insertions(+), 13 deletions(-)
create mode 100644 clang/test/OpenMP/taskgraph_target_saved_codegen.c
create mode 100644 offload/test/offloading/taskgraph_firstprivate_saved_aggregate.c
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 95191d9acff7df..f81296494e1d27 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -8519,10 +8519,21 @@ class MappableExprsHandler {
/// Function the directive is being generated for.
CodeGenFunction &CGF;
+ /// What a firstprivate clause said about one of its variables: whether the
+ /// clause was implicit, and whether it carried the "saved" modifier.
+ struct FirstPrivateInfo {
+ bool isImplicit = false;
+ bool isSaved = false;
+
+ FirstPrivateInfo() = default;
+ FirstPrivateInfo(bool isImplicit) : isImplicit(isImplicit) {}
+ FirstPrivateInfo(bool isImplicit, bool isSaved)
+ : isImplicit(isImplicit), isSaved(isSaved) {}
+ };
+
/// Set of all first private variables in the current directive.
- /// bool data is set to true if the variable is implicitly marked as
- /// firstprivate, false otherwise.
- llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls;
+ llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, FirstPrivateInfo>
+ FirstPrivateDecls;
/// Set of defaultmap clause kinds that use firstprivate behavior.
llvm::SmallSet<OpenMPDefaultmapClauseKind, 4> DefaultmapFirstprivateKinds;
@@ -9841,11 +9852,18 @@ class MappableExprsHandler {
// 'private ptr' and 'map to' flag. Return the right flags if the captured
// declaration is known as first-private in this handler.
if (FirstPrivateDecls.count(Cap.getCapturedVar())) {
+ OpenMPOffloadMappingFlags Bits;
if (Cap.getCapturedVar()->getType()->isAnyPointerType())
- return OpenMPOffloadMappingFlags::OMP_MAP_TO |
+ Bits = OpenMPOffloadMappingFlags::OMP_MAP_TO |
OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;
- return OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
- OpenMPOffloadMappingFlags::OMP_MAP_TO;
+ else
+ Bits = OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |
+ OpenMPOffloadMappingFlags::OMP_MAP_TO;
+ auto FP =
+ FirstPrivateDecls.find(Cap.getCapturedVar()->getCanonicalDecl());
+ if (FP->getSecond().isSaved)
+ Bits |= OpenMPOffloadMappingFlags::OMP_MAP_SAVED;
+ return Bits;
}
auto I = LambdasMap.find(Cap.getCapturedVar()->getCanonicalDecl());
if (I != LambdasMap.end())
@@ -10497,12 +10515,28 @@ class MappableExprsHandler {
MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF)
: CurDir(&Dir), CGF(CGF), AttachPtrComparator(*this) {
// Extract firstprivate clause information.
+ //
+ // A variable declared inside the enclosing taskgraph region has no value
+ // in the data environment a replay starts from, which is the taskgraph
+ // construct's own, so its copy comes from the saved data environment
+ // (OpenMP 6.0 [14.3]) exactly as though the clause had said `saved`. The
+ // taskgraph's captured record is what tells the two apart: it captures
+ // every automatic variable of the enclosing environment the region uses,
+ // and nothing declared within it.
+ auto DeclaredInTaskgraph = [&CGF](const VarDecl *VD) {
+ return CGF.getOMPWithinTaskgraph() && !VD->hasGlobalStorage() &&
+ CGF.CapturedStmtInfo && !CGF.CapturedStmtInfo->lookup(VD);
+ };
for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>())
for (const auto *D : C->varlist()) {
- const ValueDecl *VD = cast<DeclRefExpr>(D)->getDecl();
- if (const auto *BD = dyn_cast<BindingDecl>(VD))
- VD = cast<VarDecl>(BD->getDecomposedDecl());
- FirstPrivateDecls.try_emplace(cast<VarDecl>(VD), C->isImplicit());
+ const ValueDecl *Decl = cast<DeclRefExpr>(D)->getDecl();
+ if (const auto *BD = dyn_cast<BindingDecl>(Decl))
+ Decl = cast<VarDecl>(BD->getDecomposedDecl());
+ const auto *VD = cast<VarDecl>(Decl);
+ FirstPrivateDecls.try_emplace(VD, C->isImplicit(),
+ C->getKind() ==
+ clang::OMPC_FIRSTPRIVATE_saved ||
+ DeclaredInTaskgraph(VD));
}
// Extract implicit firstprivates from uses_allocators clauses.
for (const auto *C : Dir.getClausesOfKind<OMPUsesAllocatorsClause>()) {
@@ -11407,7 +11441,7 @@ class MappableExprsHandler {
bool isEffectivelyFirstprivate(const VarDecl *VD, QualType Type) const {
// Check explicit firstprivate clauses (not implicit from defaultmap)
auto I = FirstPrivateDecls.find(VD);
- if (I != FirstPrivateDecls.end() && !I->getSecond())
+ if (I != FirstPrivateDecls.end() && !I->getSecond().isImplicit)
return true; // Explicit firstprivate only
// Check defaultmap(firstprivate:scalar) for scalar types
@@ -11484,7 +11518,7 @@ class MappableExprsHandler {
}
auto I = FirstPrivateDecls.find(VD);
if (I != FirstPrivateDecls.end())
- IsImplicit = I->getSecond();
+ IsImplicit = I->getSecond().isImplicit;
} else {
assert(CI.capturesVariable() && "Expected captured reference.");
const auto *PtrTy = cast<ReferenceType>(RI.getType().getTypePtr());
@@ -11515,7 +11549,7 @@ class MappableExprsHandler {
}
auto I = FirstPrivateDecls.find(VD);
if (I != FirstPrivateDecls.end())
- IsImplicit = I->getSecond();
+ IsImplicit = I->getSecond().isImplicit;
}
// Every default map produces a single argument which is a target parameter.
CombinedInfo.Types.back() |=
diff --git a/clang/test/OpenMP/taskgraph_target_saved_codegen.c b/clang/test/OpenMP/taskgraph_target_saved_codegen.c
new file mode 100644
index 00000000000000..dddee588c69796
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_target_saved_codegen.c
@@ -0,0 +1,54 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c -triple x86_64-unknown-linux-gnu -fopenmp-targets=x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// A firstprivate aggregate of a target construct in a taskgraph region carries
+// the SAVED map bit when its value belongs to the saved data environment, which
+// makes the recording snapshot its bytes. That is the case when the clause
+// says 'saved', and when the variable is declared inside the taskgraph region:
+// a replay does not run its declaration again (OpenMP 6.0 [14.3]). A
+// firstprivate of a variable in the taskgraph's enclosing data environment is
+// not saved.
+
+struct S {
+ int A[16];
+ int B;
+};
+
+void outer(void) {
+ struct S s;
+#pragma omp taskgraph
+ {
+#pragma omp target firstprivate(s)
+ { s.B = 1; }
+ }
+}
+
+void saved(void) {
+ struct S s;
+#pragma omp taskgraph
+ {
+#pragma omp target firstprivate(saved : s)
+ { s.B = 1; }
+ }
+}
+
+void inner(int n) {
+#pragma omp taskgraph
+ {
+ struct S s;
+ s.B = n;
+#pragma omp target firstprivate(s)
+ { s.B += 1; }
+ }
+}
+
+// outer: PRIVATE | TO | TARGET_PARAM.
+// CHECK: @.offload_maptypes{{[.0-9]*}} = private unnamed_addr constant [{{[0-9]+}} x i64] [i64 161{{[],]}}
+// saved, and inner without being asked: the same, and SAVED.
+// CHECK: @.offload_maptypes{{[.0-9]*}} = private unnamed_addr constant [{{[0-9]+}} x i64] [i64 65697{{[],]}}
+// CHECK: @.offload_maptypes{{[.0-9]*}} = private unnamed_addr constant [{{[0-9]+}} x i64] [i64 65697{{[],]}}
+
+#endif
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
index 99be75c7e9673b..e5e3f3d6cd05fa 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPConstants.h
@@ -256,6 +256,9 @@ enum class OpenMPOffloadMappingFlags : uint64_t {
// instead of preserving the original pointer's value. Currently only
// useful in conjunction with RETURN_PARAM.
OMP_MAP_FB_NULLIFY = 0x8000,
+ // Set on a firstprivate aggregate whose clause carried the "saved" modifier,
+ // meaning a taskgraph recording must snapshot its value at record time.
+ OMP_MAP_SAVED = 0x10000,
/// Signal that the runtime library should use args as an array of
/// descriptor_dim pointers and use args_size as dims. Used when we have
/// non-contiguous list items in target update directive
diff --git a/offload/include/omptarget.h b/offload/include/omptarget.h
index 573127406a0ed8..f20670b7941b9b 100644
--- a/offload/include/omptarget.h
+++ b/offload/include/omptarget.h
@@ -86,12 +86,37 @@ enum tgt_map_type : uint64_t {
// instead of preserving the original pointer's value. Currently only
// useful in conjunction with RETURN_PARAM.
OMP_TGT_MAPTYPE_FB_NULLIFY = 0x8000,
+ // For firstprivate values, set if the (aggregate) value has the "saved"
+ // modifier: if so, we will snapshot the pointed-to data when recording
+ // taskgraphs.
+ OMP_TGT_MAPTYPE_SAVED = 0x10000,
// descriptor for non-contiguous target-update
OMP_TGT_MAPTYPE_NON_CONTIG = 0x100000000000,
// member of struct, member given by [16 MSBs] - 1
OMP_TGT_MAPTYPE_MEMBER_OF = 0xffff000000000000
};
+#ifdef __cplusplus
+/// Map-type accessor helpers, which name the bits of a map type rather than
+/// spelling out their encoding at each use.
+namespace omptarget {
+namespace maptype {
+
+/// Private (firstprivate aggregate) variable -- not mapped, needs its own
+/// per-launch device storage.
+inline bool isPrivate(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_PRIVATE;
+}
+/// Pass-by-value literal -- the argument *is* the value (no device pointer).
+inline bool isLiteral(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_LITERAL;
+}
+/// Firstprivate aggregate whose clause carried the "saved" modifier.
+inline bool isSaved(int64_t MapType) { return MapType & OMP_TGT_MAPTYPE_SAVED; }
+} // namespace maptype
+} // namespace omptarget
+#endif // __cplusplus
+
/// Flags for offload entries.
enum OpenMPOffloadingDeclareTargetFlags {
/// Mark the entry global as having a 'link' attribute.
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index 1170618a0c2e08..4beaf4512c7764 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -685,6 +685,16 @@ EXTERN void __tgt_taskgraph_dup_kernel_args(void *Dst, void *Src,
bool SizeOnly = (Dst == nullptr);
KernelArgsTy *KernArgs = static_cast<KernelArgsTy *>(Src);
size_t N = KernArgs->NumArgs;
+ namespace mt = omptarget::maptype;
+ // A firstprivate aggregate whose clause carried the "saved" modifier: its
+ // data has to be duplicated here. We only need to do this for aggregates,
+ // because scalars or pointers are passed by value.
+ auto FpAggregateP = [&](size_t Idx) {
+ int64_t ArgType = KernArgs->ArgTypes[Idx];
+ return mt::isPrivate(ArgType) && !mt::isLiteral(ArgType) &&
+ mt::isSaved(ArgType) && KernArgs->ArgPtrs[Idx] &&
+ KernArgs->ArgSizes[Idx] > 0;
+ };
// ArgNames and ArgMappers are optional; a launch with neither debug names nor
// user-defined mappers leaves them null. The copy has to preserve that, or
// every argument would appear to carry a mapper (pointing at uninitialized
@@ -712,6 +722,10 @@ EXTERN void __tgt_taskgraph_dup_kernel_args(void *Dst, void *Src,
// ArgMappers
if (HasMappers)
CountAligned(N * sizeof(void *), alignof(void *));
+ // One snapshot per saved firstprivate aggregate.
+ for (size_t I = 0; I < N; I++)
+ if (FpAggregateP(I))
+ CountAligned(KernArgs->ArgSizes[I], alignof(std::max_align_t));
*AllocSize = Total;
} else {
size_t Allocated = 0;
@@ -743,6 +757,17 @@ EXTERN void __tgt_taskgraph_dup_kernel_args(void *Dst, void *Src,
? static_cast<void **>(BumpAllocDup(
N * sizeof(void *), alignof(void *), KernArgs->ArgMappers))
: nullptr;
+ // Snapshot each saved firstprivate aggregate and redirect the argument at
+ // the copy.
+ for (size_t I = 0; I < N; I++) {
+ if (!FpAggregateP(I))
+ continue;
+ void *Saved =
+ BumpAllocDup(KernArgs->ArgSizes[I], alignof(std::max_align_t),
+ KernArgs->ArgPtrs[I]);
+ DupArgs->ArgPtrs[I] = Saved;
+ DupArgs->ArgBasePtrs[I] = Saved;
+ }
assert(Allocated == *AllocSize);
}
}
diff --git a/offload/test/offloading/taskgraph_firstprivate_saved_aggregate.c b/offload/test/offloading/taskgraph_firstprivate_saved_aggregate.c
new file mode 100644
index 00000000000000..b8f0e86f2ba7ed
--- /dev/null
+++ b/offload/test/offloading/taskgraph_firstprivate_saved_aggregate.c
@@ -0,0 +1,61 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A replayed 'omp taskgraph' with a 'target' that takes an *aggregate*
+// firstprivate(saved:) array. The saved value is the one captured when the
+// graph was recorded; mutating the host array after each encounter must not
+// affect later replays. Recording snapshots the bytes into the captured
+// kernel-arguments blob and points the argument at the snapshot, so every
+// encounter privatizes from coef[j] == j + 1 however the host copy has moved
+// on -- and it also keeps the argument off the recording invocation's stack.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define M 8
+
+int main() {
+ const int N = 256;
+ const int Iters = 3;
+
+ int coef[M];
+ for (int j = 0; j < M; ++j)
+ coef[j] = j + 1; // record-time value; sum = M*(M+1)/2 = 36.
+
+ int *acc = (int *)calloc(N, sizeof(int));
+
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target firstprivate(saved : coef) map(tofrom : acc[0 : N])
+ {
+ for (int i = 0; i < N; ++i) {
+ int s = 0;
+ for (int j = 0; j < M; ++j)
+ s += coef[j];
+ acc[i] += s;
+ }
+ }
+ }
+ // Mutate the host copy after the construct; 'saved' must make replays
+ // ignore this and keep using the record-time coef.
+ for (int j = 0; j < M; ++j)
+ coef[j] += 100;
+ }
+
+ const int Sum = M * (M + 1) / 2; // 36
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (acc[i] != Iters * Sum)
+ ++errors;
+
+ // CHECK: taskgraph saved firstprivate aggregate errors: 0
+ printf("taskgraph saved firstprivate aggregate errors: %d\n", errors);
+ free(acc);
+ return errors != 0;
+}
>From 6c41e2039246b973d4a35696bcfacad95d9e6df8 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 3 Sep 2026 05:37:45 -0500
Subject: [PATCH 17/29] [Clang][OpenMP] Relocation of target constructs in
taskgraph
This is the target construct equivalent of "Initialisation and relocation
of host-based tasks in taskgraph" from earlier in this series.
Target kernels and data mapping operations (enter data, exit data,
update) take copies of the kernel arguments or mapping arrays they are
recorded with. On a subsequent replay, just as with host tasks, these
may contain dangling pointers to the recorded taskgraph's stack frame
(not necessarily even at the same address as the current stack frame),
or may refer to non-saved firstprivate values from the point of recording
the taskgraph which may now have changed.
The approach for handling this is the same as for host tasks, though the
details are different because the data operated on is a different shape
for target constructs. We generate a relocation function in one of two
forms: the first is for kernels (relocating kernel args), the second
for data mapping constructs (relocating mapping arrays, specifically
'args_base' and 'args').
Target constructs must deal with another kind of relocation also: for
non-unified memory systems, the mappings between host addresses and
target addresses may also change between replays, and that may matter
for some target-specific graph finalization. That is dealt with by
later patches in this series, not here.
New tests have been added to cover new functionality.
Assisted-By: Claude
---
clang/lib/CodeGen/CGOpenMPRuntime.cpp | 706 +++++++++++++++++-
clang/lib/CodeGen/CGOpenMPRuntime.h | 14 +-
.../test/OpenMP/taskgraph_target_codegen.cpp | 10 +-
.../OpenMP/taskgraph_target_data_codegen.cpp | 28 +-
.../taskgraph_target_relocate_codegen.c | 376 ++++++++++
...taskgraph_target_this_relocate_codegen.cpp | 85 +++
.../test/offloading/taskgraph_target_this.cpp | 65 ++
7 files changed, 1249 insertions(+), 35 deletions(-)
create mode 100644 clang/test/OpenMP/taskgraph_target_relocate_codegen.c
create mode 100644 clang/test/OpenMP/taskgraph_target_this_relocate_codegen.cpp
create mode 100644 offload/test/offloading/taskgraph_target_this.cpp
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index f81296494e1d27..d2464dbc1b215c 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -11184,10 +11184,15 @@ class MappableExprsHandler {
// entry.
if (PartialStruct.Base.isValid()) {
CurCaptureVarInfo.append(PartialStruct.PreliminaryMapData);
+ // The combined entry is the struct itself, so it is named after
+ // the captured variable rather than left anonymous. Taskgraph
+ // relocation needs it: the entry's base pointer is the struct's
+ // address, and without a declaration to reach it by there is no
+ // way to re-derive that address on a replay from another frame.
emitCombinedEntry(
CurCaptureVarInfo, CurInfoForComponentLists.Types,
- PartialStruct, AttachInfo, Cap->capturesThis(), OMPBuilder,
- /*VD=*/nullptr, OffsetForMemberOfFlag,
+ PartialStruct, AttachInfo, Cap->capturesThis(), OMPBuilder, VD,
+ OffsetForMemberOfFlag,
/*NotTargetParams*/ !IsEligibleForTargetParamFlag);
}
@@ -11517,8 +11522,16 @@ class MappableExprsHandler {
CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty));
}
auto I = FirstPrivateDecls.find(VD);
- if (I != FirstPrivateDecls.end())
+ if (I != FirstPrivateDecls.end()) {
IsImplicit = I->getSecond().isImplicit;
+ // Tells a taskgraph's relocation helper to leave the recorded value
+ // alone rather than re-read the variable at replay.
+ if (I->getSecond().isSaved &&
+ static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
+ CombinedInfo.Types.back() &
+ OpenMPOffloadMappingFlags::OMP_MAP_LITERAL))
+ CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_SAVED;
+ }
} else {
assert(CI.capturesVariable() && "Expected captured reference.");
const auto *PtrTy = cast<ReferenceType>(RI.getType().getTypePtr());
@@ -12155,10 +12168,638 @@ emitClauseForBareTargetDirective(CodeGenFunction &CGF,
}
}
+/// Where a recorded map slot's base pointer comes from, expressed against the
+/// variables the enclosing taskgraph captures.
+///
+/// A map array records two addresses per slot, and the distance between them
+/// is a property of the *shape* of the map -- which array element the section
+/// starts at, which member of the struct it names -- rather than of the frame
+/// the encounter ran in. So relocation only has to re-derive the base and can
+/// take the distance from the recorded pair, which is what lets it accept
+/// dynamic bounds and member expressions that no constant offset could
+/// describe. It is also why `saved` is what makes a graph replayable: the
+/// distance is frozen at record time, so a bound that moves between replays is
+/// a shape change, and shapes are graph invariants.
+namespace {
+/// How the base pointer recorded for a slot is reached from the capture it
+/// belongs to. Which of the three applies is a property of the *encoding* the
+/// map entry was emitted with rather than of the expression, because the same
+/// expression is recorded differently depending on what the runtime will do
+/// with it -- see mapBaseOf.
+enum class BaseProjectionTy {
+ /// The base is the capture's own address. A whole variable, an array
+ /// section over an array, and a struct member all record the enclosing
+ /// object's address, the distance to the mapped begin being what the
+ /// recorded pair holds.
+ CaptureAddr,
+ /// The base is the address of a pointer object, which the runtime loads for
+ /// itself. This is what PTR_AND_OBJ means, and what an attach entry
+ /// records.
+ PointerAddr,
+ /// The base is the *value* of a pointer object, loaded when the entry was
+ /// emitted. Such a base moves when the pointer is assigned and not only
+ /// when the frame changes, which is why it is reloaded here.
+ PointerValue,
+};
+
+struct MapBaseTy {
+ /// The captured variable the base is reached from. Null when it is reached
+ /// from `this` instead.
+ const VarDecl *Capture = nullptr;
+ /// The base is reached from `this`: it is the object `this` points to, or a
+ /// member of it, which is what a member named in a member function's map
+ /// clause is reached through. The taskgraph's record holds `this` by value,
+ /// so its field *is* the pointer object, and the projection is always
+ /// PointerValue.
+ bool IsThis = false;
+ /// Byte offset from the capture to the pointer object. Only meaningful for
+ /// the pointer projections; a capture-address base is the capture itself and
+ /// any offset from it belongs to the recorded distance.
+ CharUnits Offset = CharUnits::Zero();
+ BaseProjectionTy Projection = BaseProjectionTy::CaptureAddr;
+};
+
+/// One recorded map slot and how to refresh it.
+struct RelocatedSlotTy {
+ enum KindTy {
+ /// Both addresses are refreshed: the base from the capture, the begin
+ /// pointer from the base plus the recorded distance.
+ Whole,
+ /// Only the base is refreshed. A non-contiguous entry's begin pointer is
+ /// not an address in the mapped data at all but a pointer to the stride
+ /// descriptor, which the runtime owns and copies at record time.
+ BaseOnly,
+ /// An attach entry's two addresses live in different objects -- the base
+ /// is the pointer itself, the begin pointer is where it points -- so the
+ /// recorded distance between them means nothing. The pointee address is
+ /// taken from the map entry the attach belongs to instead.
+ Attach,
+ };
+
+ /// Position in the recorded base-pointer / pointer arrays.
+ unsigned Index;
+ /// The capture's field in the enclosing taskgraph's record.
+ const FieldDecl *Field;
+ MapBaseTy Base;
+ KindTy Kind;
+ /// For Attach, the entry in this list whose refreshed begin pointer this
+ /// slot shares.
+ unsigned Pointee = 0;
+};
+} // namespace
+
+/// The captured variable \p E names and the byte offset of \p E within it, for
+/// the shapes whose offset is a matter of record layout rather than of a value:
+/// the variable itself, and direct member accesses on it.
+///
+/// A dereference on the way is what makes an offset a runtime question, and is
+/// declined here rather than followed, so a base reached through two pointers
+/// (`ps->p[0:n]`) declines instead of relocating from a pointer this replay has
+/// not reloaded. A reference member is a hidden dereference and goes the same
+/// way, as does a bit-field, which has no byte offset to give.
+static const VarDecl *captureOffsetOf(ASTContext &C, const Expr *E,
+ CharUnits &Offset) {
+ E = E->IgnoreParenImpCasts();
+ if (const auto *DRE = dyn_cast<DeclRefExpr>(E))
+ return dyn_cast<VarDecl>(DRE->getDecl());
+
+ const auto *ME = dyn_cast<MemberExpr>(E);
+ if (!ME || ME->isArrow())
+ return nullptr;
+ const auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl());
+ if (!FD || FD->isBitField() || FD->getType()->isReferenceType())
+ return nullptr;
+ const VarDecl *VD = captureOffsetOf(C, ME->getBase(), Offset);
+ if (!VD)
+ return nullptr;
+ Offset += C.toCharUnitsFromBits(C.getASTRecordLayout(FD->getParent())
+ .getFieldOffset(FD->getFieldIndex()));
+ return VD;
+}
+
+/// Whether \p E is `this` itself. A conversion to a base class's `this` is
+/// not: the object it points to starts somewhere else.
+static bool isThisPointer(const Expr *E) {
+ E = E->IgnoreParens();
+ while (const auto *ICE = dyn_cast<ImplicitCastExpr>(E)) {
+ if (ICE->getCastKind() != CK_NoOp)
+ break;
+ E = ICE->getSubExpr()->IgnoreParens();
+ }
+ return isa<CXXThisExpr>(E);
+}
+
+/// Where the base pointer recorded for \p MapExpr came from.
+///
+/// Two questions, answered from two different places. *Which* object the base
+/// is reached from comes from walking the map expression: inwards until either
+/// a capture, or the pointer whose pointee the map names -- an array section or
+/// subscript over a pointer, an arrow, or an explicit `*`. A `MEMBER_OF`
+/// slot's declaration is the field, so it is the expression and not the mapped
+/// declaration that leads to the enclosing struct's capture. A member named
+/// in a member function is reached through `this` in the same way (see
+/// MapBaseTy::IsThis).
+///
+/// *How* the base is reached from there is not a property of the expression at
+/// all, and cannot be inferred from it: the same `map(p[0:n])` records the
+/// pointer's address when the runtime is going to load it (`PTR_AND_OBJ`) and
+/// the loaded value when it is not, and a pointer captured implicitly by a
+/// `target` region records the loaded value from no expression whatsoever. So
+/// the projection is a matter of how the entry was emitted: \p BaseIsPointer
+/// says the base names a pointer object rather than the mapped object, and
+/// \p RuntimeLoadsBase says the runtime will dereference it itself.
+static std::optional<MapBaseTy>
+mapBaseOf(ASTContext &C, const ValueDecl *MapDecl, const Expr *MapExpr,
+ bool BaseIsPointer, bool RuntimeLoadsBase) {
+ const bool HasAttachPtr = BaseIsPointer;
+ const BaseProjectionTy PtrProjection = RuntimeLoadsBase
+ ? BaseProjectionTy::PointerAddr
+ : BaseProjectionTy::PointerValue;
+
+ // The pointer object is named directly, so its offset within the capture is
+ // part of reaching the base rather than part of the recorded distance.
+ auto pointerObject = [&](const Expr *PtrExpr) -> std::optional<MapBaseTy> {
+ MapBaseTy Base;
+ if (isThisPointer(PtrExpr)) {
+ // `this` is a value, not an object the runtime could load it from.
+ if (RuntimeLoadsBase)
+ return std::nullopt;
+ Base.IsThis = true;
+ Base.Projection = BaseProjectionTy::PointerValue;
+ return Base;
+ }
+ Base.Projection = PtrProjection;
+ Base.Capture = captureOffsetOf(C, PtrExpr, Base.Offset);
+ if (!Base.Capture)
+ return std::nullopt;
+ return Base;
+ };
+
+ // No dereference in the expression. With an attach pointer the expression
+ // *is* that pointer -- a whole-pointer map -- and the base is what it holds;
+ // without one the base is the enclosing object's address.
+ auto wholeObject = [&](const Expr *E,
+ const VarDecl *Capture) -> std::optional<MapBaseTy> {
+ if (HasAttachPtr) {
+ // Only a whole-pointer map can carry an attach pointer here: the
+ // expression is the pointer itself. A non-pointer reached without a
+ // dereference has none, so decline rather than guess.
+ if (!E->getType()->isAnyPointerType())
+ return std::nullopt;
+ return pointerObject(E);
+ }
+ MapBaseTy Base;
+ Base.Capture = Capture;
+ if (!Base.Capture)
+ return std::nullopt;
+ return Base;
+ };
+
+ // A default capture map has no expression to walk; it names the whole
+ // variable, and if the base is a pointer it is that variable.
+ if (!MapExpr) {
+ const auto *VD = dyn_cast_or_null<VarDecl>(MapDecl);
+ if (!VD)
+ return std::nullopt;
+ MapBaseTy Base;
+ Base.Capture = VD;
+ if (HasAttachPtr)
+ Base.Projection = PtrProjection;
+ return Base;
+ }
+
+ for (const Expr *E = MapExpr->IgnoreParenImpCasts();;) {
+ if (const auto *DRE = dyn_cast<DeclRefExpr>(E))
+ return wholeObject(MapExpr, dyn_cast<VarDecl>(DRE->getDecl()));
+ if (const auto *OASE = dyn_cast<ArraySectionExpr>(E)) {
+ const Expr *B = OASE->getBase()->IgnoreParenImpCasts();
+ if (ArraySectionExpr::getBaseOriginalType(B)
+ .getCanonicalType()
+ ->isAnyPointerType())
+ return pointerObject(B);
+ E = B;
+ continue;
+ }
+ if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(E)) {
+ const Expr *B = ASE->getBase()->IgnoreParenImpCasts();
+ if (B->getType()->isAnyPointerType())
+ return pointerObject(B);
+ E = B;
+ continue;
+ }
+ if (const auto *ME = dyn_cast<MemberExpr>(E)) {
+ if (ME->isArrow())
+ return pointerObject(ME->getBase());
+ if (ME->getType()->isAnyPointerType() && HasAttachPtr)
+ return pointerObject(ME);
+ E = ME->getBase()->IgnoreParenImpCasts();
+ continue;
+ }
+ if (const auto *UO = dyn_cast<UnaryOperator>(E);
+ UO && UO->getOpcode() == UO_Deref)
+ return pointerObject(UO->getSubExpr());
+ return std::nullopt;
+ }
+}
+
+/// Emit a helper with one of the runtime target-relocation signatures,
+/// kmp_target_kernel_relocate_t for a target region: void relocate(void
+/// *kernel_args, void *outer_captures); or, if \p IsDataNode,
+/// kmp_target_data_relocate_t for a target-data directive: void relocate(void
+/// **args_base, void **args, void *outer_captures);
+///
+/// A target region's recorded __tgt_kernel_arguments blob holds the two map
+/// arrays of addresses, and a target-data directive's are handed over as they
+/// are. The two differ only in where the arrays are found; everything about
+/// which slots move and how is the same, since both hold the same recorded map
+/// entries.
+///
+/// A recorded target region keeps the map arrays it was launched with, and
+/// those hold the addresses the mapped variables had on the encounter that
+/// recorded the graph. A replay from a different frame -- the same function
+/// called at another stack depth, or a different caller entirely -- finds those
+/// addresses stale. The runtime calls this helper first, with the recorded
+/// kernel-arguments blob and the taskgraph record reconstructed for *this*
+/// encounter, and the helper rewrites each stale slot.
+///
+/// A slot is rewritten by re-deriving its base pointer from the capture it came
+/// from (see MapBaseTy) and moving its begin pointer by the same distance the
+/// recorded pair held. Slots that cannot move, or must not, are skipped rather
+/// than rewritten: a by-value argument is a value and not an address, a
+/// variable with static storage duration has the same link-time-fixed address
+/// on every replay, and a firstprivate aggregate's slot points into the
+/// recorded snapshot on purpose. A slot that is neither skippable nor
+/// recognised makes the whole helper null, because rewriting some of a region's
+/// slots and leaving the rest stale is worse than declining: the caller passes
+/// null on to the runtime, which keeps its existing refuse-to-relocate
+/// behaviour.
+///
+/// A by-value (literal) slot has no address to move, but it does have a value
+/// to refresh: a firstprivate of a variable in the taskgraph's enclosing data
+/// environment is initialized from the variable's value at each replay (OpenMP
+/// 6.0 [14.3]), so the helper re-reads the variable and writes it into the
+/// slot. A `saved` list item, a variable declared inside the region and a
+/// clause's captured expression keep their recorded values, as the saved data
+/// environment requires.
+///
+/// When every slot is skippable the helper is still emitted, as a non-null
+/// no-op, the same convention emitTaskRelocationFunction uses: null means
+/// "cannot relocate", not "nothing to relocate".
+static llvm::Function *emitTargetRelocationFunction(
+ CodeGenModule &CGM, SourceLocation Loc, const CapturedStmt *CS,
+ const CodeGenFunction::CGCapturedStmtInfo *OuterCSI,
+ const MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,
+ bool IsDataNode) {
+ ASTContext &C = CGM.getContext();
+ using OpenMPOffloadMappingFlags = llvm::omp::OpenMPOffloadMappingFlags;
+
+ auto Flagged = [&](unsigned I, OpenMPOffloadMappingFlags Mask) {
+ return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
+ CombinedInfo.Types[I] & Mask) != 0;
+ };
+
+ // A pointer the region only uses -- with no map clause naming it -- is
+ // captured by copy, and generateDefaultMapInfo records its *value* as both
+ // base and begin. Nothing in the entry says so: it has no map expression to
+ // read a dereference from, and HasAttachPtr is hardcoded false there. So it
+ // is asked of the capture list, which is the only place that distinguishes a
+ // by-copy capture from a by-reference one. A data directive has no capture
+ // list and needs none: every entry of one comes from an explicit clause, so
+ // there is always a map expression to read.
+ auto capturedByCopy = [CS](const ValueDecl *VD) {
+ if (!CS)
+ return false;
+ for (const CapturedStmt::Capture &Cap : CS->captures())
+ if (Cap.capturesVariableByCopy() &&
+ Cap.getCapturedVar()->getCanonicalDecl() == VD->getCanonicalDecl())
+ return true;
+ return false;
+ };
+
+ // A by-value firstprivate whose value is refreshed on replay: the variable
+ // and where to read it, the outer record field for a captured one and null
+ // for a global.
+ struct LiteralSlotTy {
+ unsigned Index;
+ const VarDecl *VD;
+ const FieldDecl *Field;
+ };
+ SmallVector<RelocatedSlotTy, 8> Slots;
+ SmallVector<LiteralSlotTy, 4> Literals;
+ const RecordDecl *OuterRD = nullptr;
+
+ // In a member function, maps of members of the object are grouped under an
+ // entry for the object `this` points to, which has neither a declaration
+ // nor an expression. It is recognised by its members, which name it as
+ // their MEMBER_OF parent and are reached from `this`.
+ llvm::SmallDenseSet<unsigned, 4> ThisStructEntries;
+ for (unsigned J = 0, E = CombinedInfo.Exprs.size(); J != E; ++J) {
+ uint64_t MemberOf =
+ static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
+ CombinedInfo.Types[J] &
+ OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) >>
+ CGM.getOpenMPRuntime().getOMPBuilder().getFlagMemberOffset();
+ if (!MemberOf || MemberOf > E)
+ continue;
+ const unsigned Parent = MemberOf - 1;
+ const Expr *MemberExpr = CombinedInfo.Exprs[J].getMapExpr();
+ if (CombinedInfo.Exprs[Parent].getMapDecl() ||
+ CombinedInfo.Exprs[Parent].getMapExpr() || !MemberExpr)
+ continue;
+ std::optional<MapBaseTy> MemberBase =
+ mapBaseOf(C, CombinedInfo.Exprs[J].getMapDecl(), MemberExpr,
+ /*BaseIsPointer=*/false, /*RuntimeLoadsBase=*/false);
+ if (MemberBase && MemberBase->IsThis)
+ ThisStructEntries.insert(Parent);
+ }
+
+ for (unsigned I = 0, E = CombinedInfo.Exprs.size(); I != E; ++I) {
+ // A by-value argument carries the value itself, so there is no address to
+ // move. But a firstprivate of a variable in the taskgraph's enclosing data
+ // environment takes the variable's value at each replay (OpenMP 6.0
+ // [14.3]), so that value is re-read. Only an argument the region captured
+ // by copy holds the variable's bytes; the other literal entries -- a
+ // lambda's captures, VLA bounds, the trailing dyn_ptr -- hold addresses or
+ // values of expressions, and keep what the recording gave them. So do a
+ // `saved` list item (whose entry carries SAVED), a variable declared inside
+ // the region (which the taskgraph record does not capture) and a clause's
+ // captured expression.
+ if (Flagged(I, OpenMPOffloadMappingFlags::OMP_MAP_LITERAL)) {
+ const auto *VD =
+ dyn_cast_or_null<VarDecl>(CombinedInfo.Exprs[I].getMapDecl());
+ if (IsDataNode || !VD || isa<OMPCapturedExprDecl>(VD) ||
+ !Flagged(I, OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM) ||
+ Flagged(I, OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |
+ OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |
+ OpenMPOffloadMappingFlags::OMP_MAP_SAVED) ||
+ !capturedByCopy(VD) ||
+ C.getTypeSizeInChars(VD->getType().getNonReferenceType()) >
+ C.getTypeSizeInChars(C.VoidPtrTy))
+ continue;
+ const FieldDecl *Field = nullptr;
+ if (!VD->hasGlobalStorage()) {
+ Field = OuterCSI ? OuterCSI->lookup(VD) : nullptr;
+ if (!Field)
+ continue;
+ OuterRD = cast<RecordDecl>(Field->getDeclContext());
+ }
+ Literals.push_back({I, VD, Field});
+ continue;
+ }
+ // A `saved` firstprivate aggregate's slot is deliberately *not* the live
+ // variable: __tgt_taskgraph_dup_kernel_args repoints it at a snapshot
+ // inside the recorded blob, which is what makes it saved. So it is
+ // skipped, not declined -- rewriting it back to the live variable would
+ // undo exactly that, but declining the region around it would leave the
+ // ordinary maps beside it stale for no reason.
+ //
+ // Without `saved` the slot does still name the live variable, and a
+ // privatised argument is read from it at every replay, so it is relocated
+ // like any other.
+ if (Flagged(I, OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE) &&
+ Flagged(I, OpenMPOffloadMappingFlags::OMP_MAP_SAVED))
+ continue;
+
+ const ValueDecl *MapDecl = CombinedInfo.Exprs[I].getMapDecl();
+ const bool IsAttach = Flagged(I, OpenMPOffloadMappingFlags::OMP_MAP_ATTACH);
+ const Expr *MapExpr = CombinedInfo.Exprs[I].getMapExpr();
+ std::optional<MapBaseTy> Base;
+ if (ThisStructEntries.contains(I)) {
+ Base.emplace();
+ Base->IsThis = true;
+ Base->Projection = BaseProjectionTy::PointerValue;
+ } else if (MapDecl || MapExpr) {
+ Base = mapBaseOf(
+ C, MapDecl, MapExpr,
+ // An attach entry's base *is* the pointer object, which is the one
+ // thing HasAttachPtr does not say of it: the flag marks the entries
+ // that were reached *through* a pointer, and an attachment is the
+ // entry that fixes that pointer up.
+ /*BaseIsPointer=*/IsAttach || CombinedInfo.HasAttachPtr[I] ||
+ (!MapExpr && capturedByCopy(MapDecl) &&
+ MapDecl->getType()->isAnyPointerType()),
+ /*RuntimeLoadsBase=*/
+ IsAttach ||
+ Flagged(I, OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ));
+ if (!Base)
+ return nullptr;
+ } else {
+ continue;
+ }
+ // A variable with static storage duration is at the same address on every
+ // replay, so its slots are already right. That covers a pointee base
+ // through a global too: what a global pointer holds is program state, and
+ // a replay is no more entitled to reload it than to re-evaluate any other
+ // expression the graph froze.
+ if (!Base->IsThis && Base->Capture->hasGlobalStorage())
+ continue;
+ const FieldDecl *Field = nullptr;
+ if (OuterCSI)
+ Field = Base->IsThis ? OuterCSI->getThisFieldDecl()
+ : OuterCSI->lookup(Base->Capture);
+ if (!Field)
+ return nullptr;
+ OuterRD = cast<RecordDecl>(Field->getDeclContext());
+
+ RelocatedSlotTy Slot{I, Field, *Base, RelocatedSlotTy::Whole, 0};
+ if (Flagged(I, OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG)) {
+ Slot.Kind = RelocatedSlotTy::BaseOnly;
+ } else if (IsAttach) {
+ // The pointee address an attach entry publishes is the same one the map
+ // entry for the pointee data holds, so it is taken from there rather
+ // than re-derived: the two were emitted from one address, and computing
+ // it twice would be two chances to disagree. Both entries reach the
+ // pointer object by the same route, which is what identifies the pair.
+ const auto *Sibling = llvm::find_if(Slots, [&](const RelocatedSlotTy &S) {
+ return S.Kind == RelocatedSlotTy::Whole &&
+ S.Base.Projection != BaseProjectionTy::CaptureAddr &&
+ S.Base.Capture == Base->Capture &&
+ S.Base.IsThis == Base->IsThis && S.Base.Offset == Base->Offset;
+ });
+ if (Sibling == Slots.end())
+ return nullptr;
+ Slot.Kind = RelocatedSlotTy::Attach;
+ Slot.Pointee = Sibling - Slots.begin();
+ } else if (Slot.Base.Projection == BaseProjectionTy::PointerAddr) {
+ // A `PTR_AND_OBJ` data entry records the pointer's *address* as its base
+ // and its pointee's address as its begin, so the recorded pair is not a
+ // distance within one object and there is nothing to carry over: the
+ // distance that matters is from the pointer's old *value*, which the
+ // recording did not keep. Clang records the loaded value instead and
+ // leaves the loading to the attach entry, so no such entry is expected;
+ // were one to arrive, declining is the only safe answer.
+ assert(false && "PTR_AND_OBJ data entry in a recorded target region");
+ return nullptr;
+ }
+ Slots.push_back(Slot);
+ }
+
+ // void relocate(void *kernel_args, void *outer_captures) for a target
+ // region, or
+ // void relocate(void **args_base, void **args, void *outer_captures)
+ // for a data directive.
+ auto MakeArg = [&](QualType Ty) {
+ return ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, Ty,
+ ImplicitParamKind::Other);
+ };
+ FunctionArgList Args;
+ ImplicitParamDecl *KernelArgsArg = nullptr;
+ ImplicitParamDecl *BasePtrsArg = nullptr;
+ ImplicitParamDecl *PtrsArg = nullptr;
+ if (IsDataNode) {
+ BasePtrsArg = MakeArg(C.getPointerType(C.VoidPtrTy));
+ PtrsArg = MakeArg(C.getPointerType(C.VoidPtrTy));
+ Args.append({BasePtrsArg, PtrsArg});
+ } else {
+ KernelArgsArg = MakeArg(C.VoidPtrTy);
+ Args.push_back(KernelArgsArg);
+ }
+ auto *OuterArg = MakeArg(C.VoidPtrTy);
+ Args.push_back(OuterArg);
+ QualType RetTy = C.VoidTy;
+ const CGFunctionInfo &FnInfo =
+ CGM.getTypes().arrangeBuiltinFunctionDeclaration(RetTy, Args);
+
+ std::string Name = CGM.getOpenMPRuntime().getName(
+ {"omp", "taskgraph", "target", "relocate", ""});
+ auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(FnInfo),
+ llvm::GlobalValue::InternalLinkage, Name,
+ &CGM.getModule());
+ CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
+ if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
+ Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
+ Fn->setDoesNotRecurse();
+
+ CodeGenFunction CGF(CGM);
+ CGF.StartFunction(GlobalDecl(), RetTy, Fn, FnInfo, Args, Loc, Loc);
+
+ if (!Slots.empty() || !Literals.empty()) {
+ CGBuilderTy &Bld = CGF.Builder;
+ CharUnits PtrAlign = CGF.getPointerAlign();
+
+ // The two arrays of addresses this helper refreshes. A data directive
+ // hands them over as they are; a target region hands over its recorded
+ // __tgt_kernel_arguments, where their field numbers are fixed by the
+ // KernelArgs struct in OMPKinds.def.
+ auto LoadArg = [&](const ImplicitParamDecl *Arg) {
+ return Address(Bld.CreateLoad(CGF.GetAddrOfLocalVar(Arg)), CGF.VoidPtrTy,
+ PtrAlign);
+ };
+ Address BasePtrs = Address::invalid();
+ Address Ptrs = Address::invalid();
+ if (IsDataNode) {
+ BasePtrs = LoadArg(BasePtrsArg);
+ Ptrs = LoadArg(PtrsArg);
+ } else {
+ Address KernelArgs(Bld.CreateLoad(CGF.GetAddrOfLocalVar(KernelArgsArg)),
+ CGM.getOpenMPRuntime().getOMPBuilder().KernelArgs,
+ PtrAlign);
+ auto LoadArray = [&](unsigned Field) {
+ return Address(Bld.CreateLoad(Bld.CreateStructGEP(KernelArgs, Field)),
+ CGF.VoidPtrTy, PtrAlign);
+ };
+ BasePtrs = LoadArray(2);
+ Ptrs = LoadArray(3);
+ }
+
+ // Base of the taskgraph record reconstructed for this replay. There is
+ // none to read when the only slots are literals of globals.
+ LValue OuterBase;
+ if (OuterRD) {
+ QualType OuterTy = C.getCanonicalTagType(OuterRD);
+ OuterBase = CGF.MakeAddrLValue(
+ Address(Bld.CreateLoad(CGF.GetAddrOfLocalVar(OuterArg)),
+ CGF.ConvertTypeForMem(OuterTy), PtrAlign),
+ OuterTy);
+ }
+
+ // Kept so an attach slot can take the pointee address its sibling arrived
+ // at. The reads below all precede their own slot's writes, and no two
+ // slots share an index, so the recorded pair a slot reads is still the one
+ // the recording left there.
+ SmallVector<llvm::Value *, 8> NewPointers(Slots.size(), nullptr);
+
+ for (const auto &[N, Slot] : llvm::enumerate(Slots)) {
+ // EmitLValueForField auto-loads the outer reference field, so this is
+ // the variable's live address rather than the record slot holding it.
+ // `this` is held by value, so for it this is the slot, which is the
+ // pointer object its base is loaded from.
+ llvm::Value *Var =
+ CGF.EmitLValueForField(OuterBase, Slot.Field).getPointer(CGF);
+ Var = Bld.CreatePointerBitCastOrAddrSpaceCast(Var, CGM.VoidPtrTy);
+
+ // Where the pointer object sits, for a base reached through one.
+ llvm::Value *PtrObj =
+ Slot.Base.Offset.isZero()
+ ? Var
+ : Bld.CreateConstInBoundsGEP1_64(CGF.Int8Ty, Var,
+ Slot.Base.Offset.getQuantity());
+ llvm::Value *NewBase = Var;
+ switch (Slot.Base.Projection) {
+ case BaseProjectionTy::CaptureAddr:
+ break;
+ case BaseProjectionTy::PointerAddr:
+ NewBase = PtrObj;
+ break;
+ case BaseProjectionTy::PointerValue:
+ NewBase = Bld.CreateAlignedLoad(CGM.VoidPtrTy, PtrObj, PtrAlign);
+ break;
+ }
+
+ Address BaseSlot = Bld.CreateConstGEP(BasePtrs, Slot.Index);
+ Address PtrSlot = Bld.CreateConstGEP(Ptrs, Slot.Index);
+
+ if (Slot.Kind == RelocatedSlotTy::Whole) {
+ llvm::Value *OldBase = Bld.CreateLoad(BaseSlot);
+ llvm::Value *OldPtr = Bld.CreateLoad(PtrSlot);
+ NewPointers[N] =
+ Bld.CreateGEP(CGF.Int8Ty, NewBase,
+ Bld.CreatePtrDiff(CGF.Int8Ty, OldPtr, OldBase));
+ } else if (Slot.Kind == RelocatedSlotTy::Attach) {
+ NewPointers[N] = NewPointers[Slot.Pointee];
+ }
+
+ Bld.CreateStore(NewBase, BaseSlot);
+ if (NewPointers[N])
+ Bld.CreateStore(NewPointers[N], PtrSlot);
+ }
+
+ // A literal's slots hold the variable's bytes at the bottom of a
+ // pointer-sized word, the way the by-copy capture put them there, and the
+ // rest of the word is whatever the capture's temporary held.
+ for (const LiteralSlotTy &L : Literals) {
+ QualType Ty = L.VD->getType().getNonReferenceType();
+ LValue Src;
+ if (L.Field) {
+ Src = CGF.EmitLValueForField(OuterBase, L.Field);
+ } else {
+ DeclRefExpr DRE(C, const_cast<VarDecl *>(L.VD),
+ /*RefersToEnclosingVariableOrCapture=*/false, Ty,
+ VK_LValue, Loc);
+ Src = CGF.EmitLValue(&DRE);
+ }
+ llvm::Type *BitsTy = CGF.Builder.getIntNTy(
+ static_cast<unsigned>(C.getTypeSizeInChars(Ty).getQuantity()) * 8);
+ llvm::Value *New =
+ Bld.CreateLoad(Src.getAddress().withElementType(BitsTy));
+ Address BaseSlot =
+ Bld.CreateConstGEP(BasePtrs, L.Index).withElementType(BitsTy);
+ Address PtrSlot =
+ Bld.CreateConstGEP(Ptrs, L.Index).withElementType(BitsTy);
+ Bld.CreateStore(New, BaseSlot);
+ Bld.CreateStore(New, PtrSlot);
+ }
+ }
+
+ CGF.FinishFunction();
+ return Fn;
+}
+
void CGOpenMPRuntime::emitTaskgraphTargetCall(
CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc,
llvm::Value *OutlinedFnID, llvm::Value *DeviceID, llvm::Value *NumTeams,
- llvm::Value *NumThreads, llvm::Value *KernelArgsPtr) {
+ llvm::Value *NumThreads, llvm::Value *KernelArgsPtr,
+ llvm::Value *Relocate) {
llvm::Value *ThreadID = getThreadID(CGF, Loc);
llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
@@ -12182,12 +12823,6 @@ void CGOpenMPRuntime::emitTaskgraphTargetCall(
llvm::Value *HasNoWait =
CGF.Builder.getInt32(D.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
- // Relocation callback used at replay to rewrite moved host pointers in the
- // captured kernel arguments. Emission of a real relocate function is not yet
- // implemented (the recorded graph is not replayed through this path yet), so
- // a null callback is passed; the runtime simply stores it.
- llvm::Value *Relocate = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
-
llvm::Value *Args[] = {UpLoc, ThreadID, NumDeps, DepList,
HasNoWait, DeviceID, NumTeams, NumThreads,
OutlinedFnID, KernelArgsPtr, Relocate};
@@ -12201,7 +12836,8 @@ void CGOpenMPRuntime::emitTaskgraphTargetDataCall(
llvm::Value *DeviceID, unsigned NumTargetItems,
llvm::Value *BasePointersArray, llvm::Value *PointersArray,
llvm::Value *SizesArray, llvm::Value *MapTypesArray,
- llvm::Value *MapNamesArray, llvm::Value *MappersArray) {
+ llvm::Value *MapNamesArray, llvm::Value *MappersArray,
+ llvm::Value *Relocate) {
llvm::Value *ThreadID = getThreadID(CGF, Loc);
llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
@@ -12241,10 +12877,6 @@ void CGOpenMPRuntime::emitTaskgraphTargetDataCall(
llvm_unreachable("Unexpected standalone target data directive.");
}
- // See emitTaskgraphTargetCall: a real relocate callback is not emitted yet,
- // so a null pointer is passed and merely stored by the runtime.
- llvm::Value *Relocate = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
-
llvm::Value *Args[] = {UpLoc, ThreadID, NumDeps,
DepList, HasNoWait, DeviceID,
PointerNum, BasePointersArray, PointersArray,
@@ -12308,7 +12940,8 @@ static void emitTargetCallKernelLaunch(
auto &&EmitBody = [&OMPRuntime, OutlinedFn, &D, &CapturedVars,
RequiresOuterTask, &CS, OffloadingMandatory, Device,
OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,
- SizeEmitter](CodeGenFunction &CGF, bool Recording) {
+ SizeEmitter, &CGM,
+ &CombinedInfo](CodeGenFunction &CGF, bool Recording) {
bool IsReverseOffloading = Device.getInt() == OMPC_DEVICE_ancestor;
if (IsReverseOffloading) {
@@ -12391,11 +13024,23 @@ static void emitTargetCallKernelLaunch(
// emitKernelArgsStruct drives the OMPIRBuilder's own IRBuilder; keep
// CGF's builder in sync so the runtime call is emitted after the stores.
CGF.Builder.restoreIP(OMPRuntime->getOMPBuilder().getInsertionPoint());
+ // A replay may run from a different frame than the encounter that
+ // recorded the region, so the recorded map arrays need refreshing before
+ // they are used again. CapturedStmtInfo here is the target region's
+ // inlined region, whose lookup chains out to the enclosing taskgraph's
+ // record -- the same record the runtime hands back as outer_captures.
+ llvm::Function *RelocateFn = emitTargetRelocationFunction(
+ CGM, D.getBeginLoc(), &CS, CGF.CapturedStmtInfo, CombinedInfo,
+ /*IsDataNode=*/false);
+ llvm::Value *Relocate =
+ RelocateFn ? cast<llvm::Value>(RelocateFn)
+ : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
// The taskgraph entry point satisfies the dependences itself; there is no
// enclosing helper task on this path.
- OMPRuntime->emitTaskgraphTargetCall(
- CGF, D, D.getBeginLoc(), OutlinedFnID, DeviceID,
- Args.NumTeams.front(), Args.NumThreads.front(), KernelArgsPtr);
+ OMPRuntime->emitTaskgraphTargetCall(CGF, D, D.getBeginLoc(), OutlinedFnID,
+ DeviceID, Args.NumTeams.front(),
+ Args.NumThreads.front(),
+ KernelArgsPtr, Relocate);
} else {
llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
cantFail(OMPRuntime->getOMPBuilder().emitKernelLaunch(
@@ -13166,9 +13811,11 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
CodeGenFunction::OMPTargetDataInfo InputInfo;
llvm::Value *MapTypesArray = nullptr;
llvm::Value *MapNamesArray = nullptr;
+ llvm::Value *Relocate = nullptr;
// Generate the code for the opening of the data environment.
auto &&EmitDataBody = [this, &D, Device, &InputInfo, &MapTypesArray,
- &MapNamesArray](CodeGenFunction &CGF, bool Recording) {
+ &MapNamesArray,
+ &Relocate](CodeGenFunction &CGF, bool Recording) {
// Emit device ID if any.
llvm::Value *DeviceID = nullptr;
if (Device) {
@@ -13188,7 +13835,7 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
InputInfo.BasePointersArray.emitRawPointer(CGF),
InputInfo.PointersArray.emitRawPointer(CGF),
InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray,
- MapNamesArray, InputInfo.MappersArray.emitRawPointer(CGF));
+ MapNamesArray, InputInfo.MappersArray.emitRawPointer(CGF), Relocate);
return;
}
@@ -13312,8 +13959,9 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
};
auto &&TargetThenGen = [this, &EmitDataBody, &D, &InputInfo, &MapTypesArray,
- &MapNamesArray, RequiresOuterTask, ReplayableCond](
- CodeGenFunction &CGF, PrePostActionTy &) {
+ &MapNamesArray, &Relocate, RequiresOuterTask,
+ ReplayableCond](CodeGenFunction &CGF,
+ PrePostActionTy &) {
// Fill up the arrays with all the mapped variables.
MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
CGOpenMPRuntime::TargetDataInfo Info;
@@ -13334,6 +13982,18 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
MapTypesArray = Info.RTArgs.MapTypesArray;
MapNamesArray = Info.RTArgs.MapNamesArray;
+ // A replay may run from a different frame than the encounter that recorded
+ // the directive, so the recorded map arrays need refreshing before they are
+ // used again. Built here rather than at the call, which is inside a lambda
+ // that no longer has the map info the decision is made from.
+ if (ReplayableCond) {
+ llvm::Function *RelocateFn = emitTargetRelocationFunction(
+ CGM, D.getBeginLoc(), /*CS=*/nullptr, CGF.CapturedStmtInfo,
+ CombinedInfo, /*IsDataNode=*/true);
+ Relocate = RelocateFn ? cast<llvm::Value>(RelocateFn)
+ : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+ }
+
// The taskgraph (recording) path is never wrapped in a hidden helper task:
// a replayable target-data directive executes synchronously and the
// taskgraph entry point owns its own dependences. The ordinary
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 5e9f639506be1f..aadc12ca2cad20 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -1495,12 +1495,17 @@ class CGOpenMPRuntime {
/// identify the region. Depend clauses of \p D are forwarded to the entry
/// point, which satisfies them synchronously; a replayable target is never
/// wrapped in a hidden helper task.
+ ///
+ /// \p Relocate is the kmp_target_kernel_relocate_t the runtime calls before
+ /// each replay to refresh map slots whose host addresses moved with the
+ /// frame, or a null pointer where the region's maps cannot be relocated.
void emitTaskgraphTargetCall(CodeGenFunction &CGF,
const OMPExecutableDirective &D,
SourceLocation Loc, llvm::Value *OutlinedFnID,
llvm::Value *DeviceID, llvm::Value *NumTeams,
llvm::Value *NumThreads,
- llvm::Value *KernelArgsPtr);
+ llvm::Value *KernelArgsPtr,
+ llvm::Value *Relocate);
/// Record a standalone 'target {enter|exit} data' / 'target update' directive
/// nested inside a taskgraph by emitting the matching
@@ -1508,12 +1513,17 @@ class CGOpenMPRuntime {
/// arrays are passed directly, and depend clauses of \p D are forwarded to
/// the entry point, which satisfies them synchronously; a replayable
/// target-data directive is never wrapped in a hidden helper task.
+ ///
+ /// \p Relocate refreshes the recorded map arrays for a replay from a
+ /// different frame, or is null where the maps cannot be re-derived; it is
+ /// built by the caller, which is where the map info the decision needs is.
void emitTaskgraphTargetDataCall(
CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc,
llvm::Value *DeviceID, unsigned NumTargetItems,
llvm::Value *BasePointersArray, llvm::Value *PointersArray,
llvm::Value *SizesArray, llvm::Value *MapTypesArray,
- llvm::Value *MapNamesArray, llvm::Value *MappersArray);
+ llvm::Value *MapNamesArray, llvm::Value *MappersArray,
+ llvm::Value *Relocate);
/// Emit the target regions enclosed in \a GD function definition or
/// the function itself in case it is a valid device function. Returns true if
diff --git a/clang/test/OpenMP/taskgraph_target_codegen.cpp b/clang/test/OpenMP/taskgraph_target_codegen.cpp
index 0e9281909ac20e..879e7003d040ce 100644
--- a/clang/test/OpenMP/taskgraph_target_codegen.cpp
+++ b/clang/test/OpenMP/taskgraph_target_codegen.cpp
@@ -41,13 +41,13 @@ void inside_depend() {
// CHECK-NOT: @__kmpc_taskgraph_target
// Inside a taskgraph: recorded via __kmpc_taskgraph_target. The trailing
-// arguments are the __tgt_kernel_arguments struct and the (currently null)
-// relocation callback; ndeps (arg 3) is 0 and the dep_list (arg 4) is null for
-// a target with no depend clause.
-// CHECK: call i32 @__kmpc_taskgraph_target(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr %kernel_args{{[0-9.]*}}, ptr null)
+// arguments are the __tgt_kernel_arguments struct and the relocation callback
+// (see taskgraph_target_relocate_codegen.c); ndeps (arg 3) is 0 and the
+// dep_list (arg 4) is null for a target with no depend clause.
+// CHECK: call i32 @__kmpc_taskgraph_target(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr %kernel_args{{[0-9.]*}}, ptr @.omp.taskgraph.target.relocate.{{[0-9.]*}})
// Inside a taskgraph with depend(out:) + nowait: ndeps is non-zero, the
// dep_list points at the emitted kmp_depend_info array, and has_no_wait is 1.
-// CHECK: call i32 @__kmpc_taskgraph_target(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 1, ptr {{[^,]+}}, i32 1, i64 {{[^,]+}}, i32 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr %kernel_args{{[0-9.]*}}, ptr null)
+// CHECK: call i32 @__kmpc_taskgraph_target(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 1, ptr {{[^,]+}}, i32 1, i64 {{[^,]+}}, i32 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr %kernel_args{{[0-9.]*}}, ptr @.omp.taskgraph.target.relocate.{{[0-9.]*}})
#endif
diff --git a/clang/test/OpenMP/taskgraph_target_data_codegen.cpp b/clang/test/OpenMP/taskgraph_target_data_codegen.cpp
index 05f8e4ef8cd37a..70e8dabf299fb1 100644
--- a/clang/test/OpenMP/taskgraph_target_data_codegen.cpp
+++ b/clang/test/OpenMP/taskgraph_target_data_codegen.cpp
@@ -35,10 +35,28 @@ void inside() {
// Inside a taskgraph: recorded via the matching taskgraph entry points. With
// no depend clause, ndeps (arg 3) is 0 and dep_list (arg 4) is null; the seven
-// map-info pointers are forwarded, followed by the (currently null) relocation
-// callback as the trailing argument.
-// CHECK: call void @__kmpc_taskgraph_target_enter_data(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr null)
-// CHECK: call void @__kmpc_taskgraph_target_update(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr null)
-// CHECK: call void @__kmpc_taskgraph_target_exit_data(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr null)
+// map-info pointers are forwarded, followed by the relocation callback as the
+// trailing argument -- one per action, since each records its own map arrays.
+// CHECK: call void @__kmpc_taskgraph_target_enter_data(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr @.omp.taskgraph.target.relocate{{[0-9.]*}})
+// CHECK: call void @__kmpc_taskgraph_target_update(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr @.omp.taskgraph.target.relocate{{[0-9.]*}})
+// CHECK: call void @__kmpc_taskgraph_target_exit_data(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr @.omp.taskgraph.target.relocate{{[0-9.]*}})
+
+// The map arrays a data action records hold the addresses the mapped variables
+// had on the encounter that recorded them, so each action gets a callback that
+// refreshes them for a replay from a different frame. It takes the recorded
+// base-pointer and pointer arrays as its first two arguments, rather than a
+// target region's kernel-args blob: a data node's two arrays are separate
+// allocations.
+// CHECK: define internal void @.omp.taskgraph.target.relocate.(ptr noundef %0, ptr noundef %1, ptr noundef %2)
+// CHECK: [[BASES:%.+]] = load ptr, ptr %.addr,
+// CHECK-NEXT: [[PTRS:%.+]] = load ptr, ptr %.addr1,
+// x is a whole-variable map, so its base is the capture's live address and the
+// recorded distance carries the begin pointer to the same place.
+// CHECK: [[X:%.+]] = load ptr, ptr {{%.+}}, align 8, !nonnull
+// CHECK-NEXT: [[XB:%.+]] = getelementptr ptr, ptr [[BASES]], i64 0
+// CHECK-NEXT: [[XP:%.+]] = getelementptr ptr, ptr [[PTRS]], i64 0
+// CHECK: store ptr [[X]], ptr [[XB]],
+// CHECK-NEXT: store ptr {{%.+}}, ptr [[XP]],
+// CHECK-NEXT: ret void
#endif
diff --git a/clang/test/OpenMP/taskgraph_target_relocate_codegen.c b/clang/test/OpenMP/taskgraph_target_relocate_codegen.c
new file mode 100644
index 00000000000000..d2270e673c33e2
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_target_relocate_codegen.c
@@ -0,0 +1,376 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c -triple x86_64-unknown-linux-gnu -fopenmp-targets=x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// A recorded 'omp target' keeps the map arrays it was launched with, so the
+// addresses in them are the ones the mapped variables had on the encounter
+// that recorded the graph. A replay from a different frame finds them stale.
+// __kmpc_taskgraph_target therefore takes a relocation callback, which the
+// runtime calls with the recorded __tgt_kernel_arguments blob and the
+// taskgraph record reconstructed for this encounter, and which rewrites the
+// base-pointer and pointer arrays in place.
+//
+// Each slot holds two addresses, and the distance between them is a property
+// of the map's shape rather than of the frame, so the callback re-derives only
+// the base pointer and moves the begin pointer by the distance it finds in the
+// recorded pair. That is what lets a dynamic bound or a struct member through:
+// the arithmetic that produced the begin pointer never has to be repeated.
+// The callback is emitted only where every slot's base is reachable from a
+// captured variable.
+
+int G[64];
+
+// Whole-variable maps: the base is the capture's live address and the recorded
+// distance is zero, so base and begin come out the same. The third slot is the
+// implicit dyn_ptr argument, which names no variable.
+void whole(void) {
+ int A[64];
+ int s = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A) map(tofrom : s)
+ { A[0] += s; }
+ }
+}
+
+// A section: the base is still the variable, and the begin pointer is the
+// variable plus whatever the recording put between them.
+void constant_section(void) {
+ int A[64];
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A[4 : 10])
+ { A[4] = 1; }
+ }
+}
+
+// A variable with static storage duration is at the same address on every
+// replay, so it is skipped rather than rewritten. Nothing else here needs
+// relocating either, and the callback is still emitted -- as a non-null no-op,
+// because a null callback means "cannot relocate", not "nothing to relocate".
+void only_global(void) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : G[0 : 64])
+ { G[0] = 1; }
+ }
+}
+
+// A pointer base: the mapped region begins at the *value* of p, so the base is
+// reloaded from the capture rather than being its address. The attach entry
+// beside it is the other way round -- its base is the pointer object itself --
+// and its pointee address is the one the map entry arrived at, since the two
+// were emitted from a single address.
+void pointer_base(int *p, int n) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : p[0 : n])
+ { p[0] = 1; }
+ }
+}
+
+// A dynamic lower bound is carried by the recorded distance, so nothing here
+// depends on i being a constant. What it does depend on is i not changing
+// between replays, which is what `saved` is for: a moved bound is a change of
+// shape, and a graph's extents and offsets are fixed when it is recorded. The
+// body's own firstprivate copy of i is another matter: it is a value, and it
+// takes i's value at the replay.
+void dynamic_bound(int i) {
+ int A[64];
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A[i : 10])
+ { A[i] = 1; }
+ }
+}
+
+// A struct member. Two slots share one base: the combined entry covering the
+// mapped part of the struct, and the member-of entry inside it.
+struct S {
+ int A[64];
+ int B;
+ int *P;
+};
+
+void struct_member(void) {
+ struct S s;
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : s.A[0 : 64])
+ { s.A[0] = 1; }
+ }
+}
+
+// A pointer reached through another pointer. Re-deriving this base would mean
+// loading `ps`, which is a value the replay has no licence to re-read: the
+// graph froze it when it recorded the map. Declined.
+void double_indirection(struct S *ps, int n) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : ps->P[0 : n])
+ { ps->P[0] = 1; }
+ }
+}
+
+// A saved firstprivate aggregate's slot is repointed at a snapshot taken when
+// the graph was recorded; rewriting it back to the live variable would undo
+// what makes it saved. Skipped, and since it is the only slot the callback is
+// a no-op -- but a non-null one, so the maps beside such an aggregate still
+// relocate.
+void saved_aggregate(void) {
+ struct S s;
+#pragma omp taskgraph
+ {
+#pragma omp target firstprivate(saved : s)
+ { s.B = 1; }
+ }
+}
+
+// A firstprivate aggregate declared inside the region has no value in the
+// environment a replay starts from -- the taskgraph construct's -- so its copy
+// comes from the saved data environment without being asked for: the entry
+// carries the saved bit exactly as saved_aggregate's does, and its slot is
+// skipped the same way.
+void inner_aggregate(int n) {
+#pragma omp taskgraph
+ {
+ struct S s;
+ s.B = n;
+#pragma omp target firstprivate(s)
+ { s.B += 1; }
+ }
+}
+
+// A scalar the region captures by copy is a literal: its slot holds the value,
+// not an address. n is in the taskgraph's enclosing data environment, so its
+// firstprivate takes n's value at the replay; the helper re-reads it through
+// the taskgraph record and writes it into both slots at n's own width. A
+// global is re-read from the global.
+int GS;
+void literal_scalar(int n) {
+ int A[64];
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A) firstprivate(GS)
+ { A[0] = n + GS; }
+ }
+}
+
+// None of these is re-read: a `saved` scalar keeps its recorded value, which
+// its entry's SAVED bit says; a scalar declared inside the region has no value
+// in the environment a replay starts from; and a clause argument's captured
+// expression is evaluated when the graph is recorded.
+void literal_kept(int n, int m) {
+#pragma omp taskgraph
+ {
+ int k = n;
+#pragma omp target teams num_teams(m) firstprivate(saved : n)
+ { k += n; }
+ }
+}
+
+// A 'target update' records map arrays of its own and gets a callback of its
+// own, which takes the two arrays of addresses as its first two arguments
+// rather than a kernel-args blob. A strided section is the one shape where the
+// recorded begin pointer is not an address in the mapped data at all but a
+// pointer to the stride descriptor, which the runtime owns and copied when the
+// graph was recorded -- so only the base is rewritten, and the descriptor's
+// offsets, being measured from it, still name the same chunks.
+void strided(void) {
+ int A[8][8];
+#pragma omp taskgraph
+ {
+#pragma omp target update to(A[1 : 2][3 : 4])
+ }
+}
+
+// Each helper is emitted just after the region it belongs to, so these run in
+// source order. CHECK-LABEL cannot be used because it forbids the variables
+// that tie a call site to the helper it names.
+
+// Map types are emitted up front, also in source order. The firstprivate
+// entry of saved_aggregate and that of inner_aggregate are both
+// PRIVATE | TO | TARGET_PARAM | SAVED.
+// CHECK: @.offload_maptypes.{{[0-9]+}} = private unnamed_addr constant [2 x i64] [i64 65697, i64 288]
+// CHECK: @.offload_maptypes.{{[0-9]+}} = private unnamed_addr constant [2 x i64] [i64 65697, i64 288]
+// literal_scalar: A, then n (LITERAL | TARGET_PARAM | IMPLICIT), GS (LITERAL |
+// TARGET_PARAM) and dyn_ptr.
+// CHECK: @.offload_maptypes.{{[0-9]+}} = private unnamed_addr constant [4 x i64] [i64 35, i64 800, i64 288, i64 288]
+// literal_kept: the inner k and the saved n are literals carrying SAVED; the
+// num_teams expression's capture is a plain literal.
+// CHECK: @.offload_maptypes.{{[0-9]+}} = private unnamed_addr constant [4 x i64] [i64 66336, i64 65824, i64 800, i64 288]
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[WHOLE:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// Field 2 of __tgt_kernel_arguments is ArgBasePtrs and field 3 is ArgPtrs; the
+// sizes and map types beside them describe the shape of the maps, which
+// relocation never changes.
+// CHECK: define internal void [[WHOLE]](ptr noundef %0, ptr noundef %1)
+// CHECK: [[KA:%.+]] = load ptr, ptr %.addr,
+// CHECK: getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr [[KA]], i32 0, i32 2
+// CHECK-NEXT: [[BASES:%.+]] = load ptr,
+// CHECK-NEXT: getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr [[KA]], i32 0, i32 3
+// CHECK-NEXT: [[PTRS:%.+]] = load ptr,
+// CHECK-NEXT: [[REC:%.+]] = load ptr, ptr %.addr1,
+// A: the base is the capture's live address, and the begin pointer is that
+// address plus the distance the recorded pair still holds.
+// CHECK-NEXT: getelementptr inbounds nuw %struct.anon, ptr [[REC]], i32 0, i32 0
+// CHECK-NEXT: [[A:%.+]] = load ptr,
+// CHECK-NEXT: [[AB:%.+]] = getelementptr ptr, ptr [[BASES]], i64 0
+// CHECK-NEXT: [[AP:%.+]] = getelementptr ptr, ptr [[PTRS]], i64 0
+// CHECK-NEXT: [[AOB:%.+]] = load ptr, ptr [[AB]],
+// CHECK-NEXT: [[AOP:%.+]] = load ptr, ptr [[AP]],
+// CHECK-NEXT: [[AE:%.+]] = ptrtoaddr ptr [[AOP]] to i64
+// CHECK-NEXT: [[ABI:%.+]] = ptrtoaddr ptr [[AOB]] to i64
+// CHECK-NEXT: [[AD:%.+]] = sub i64 [[AE]], [[ABI]]
+// CHECK-NEXT: [[ANEW:%.+]] = getelementptr i8, ptr [[A]], i64 [[AD]]
+// CHECK-NEXT: store ptr [[A]], ptr [[AB]],
+// CHECK-NEXT: store ptr [[ANEW]], ptr [[AP]],
+// s: likewise, in the next slot.
+// CHECK-NEXT: getelementptr inbounds nuw %struct.anon, ptr [[REC]], i32 0, i32 1
+// CHECK-NEXT: [[S:%.+]] = load ptr,
+// CHECK-NEXT: [[SB:%.+]] = getelementptr ptr, ptr [[BASES]], i64 1
+// CHECK-NEXT: [[SP:%.+]] = getelementptr ptr, ptr [[PTRS]], i64 1
+// CHECK: store ptr [[S]], ptr [[SB]],
+// CHECK-NEXT: store ptr {{%.+}}, ptr [[SP]],
+// The dyn_ptr slot is a literal, so it is left alone.
+// CHECK-NEXT: ret void
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[SECTION:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// CHECK: define internal void [[SECTION]](ptr noundef %0, ptr noundef %1)
+// CHECK: [[CAP:%.+]] = load ptr, ptr {{%.+}}, align 8, !nonnull
+// CHECK-NEXT: [[CB:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 0
+// CHECK-NEXT: [[CP:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 0
+// CHECK: [[CNEW:%.+]] = getelementptr i8, ptr [[CAP]], i64 {{%.+}}
+// CHECK-NEXT: store ptr [[CAP]], ptr [[CB]],
+// CHECK-NEXT: store ptr [[CNEW]], ptr [[CP]],
+// CHECK-NEXT: ret void
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[GLOBAL:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// CHECK: define internal void [[GLOBAL]](ptr noundef %0, ptr noundef %1)
+// No slot is indexed, so nothing is rewritten.
+// CHECK-NOT: getelementptr ptr,
+// CHECK: ret void
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[POINTER:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// CHECK: define internal void [[POINTER]](ptr noundef %0, ptr noundef %1)
+// The capture is the pointer object; the base is what it currently holds.
+// CHECK: [[PVAR:%.+]] = load ptr, ptr {{%.+}}, align 8, !nonnull
+// CHECK-NEXT: [[PVAL:%.+]] = load ptr, ptr [[PVAR]],
+// CHECK-NEXT: [[PB:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 0
+// CHECK-NEXT: [[PP:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 0
+// CHECK: [[PNEW:%.+]] = getelementptr i8, ptr [[PVAL]], i64 {{%.+}}
+// CHECK-NEXT: store ptr [[PVAL]], ptr [[PB]],
+// CHECK-NEXT: store ptr [[PNEW]], ptr [[PP]],
+// The attach entry: base is the pointer object itself, pointee is the address
+// the map entry above arrived at, with no distance of its own.
+// CHECK: [[AVAR:%.+]] = load ptr, ptr {{%.+}}, align 8, !nonnull
+// CHECK-NEXT: [[TB:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 1
+// CHECK-NEXT: [[TP:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 1
+// CHECK-NEXT: store ptr [[AVAR]], ptr [[TB]],
+// CHECK-NEXT: store ptr [[PNEW]], ptr [[TP]],
+// CHECK-NEXT: ret void
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[DYNAMIC:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// CHECK: define internal void [[DYNAMIC]](ptr noundef %0, ptr noundef %1)
+// CHECK: [[DVAR:%.+]] = load ptr, ptr {{%.+}}, align 8, !nonnull
+// CHECK-NEXT: [[DB:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 0
+// CHECK-NEXT: [[DP:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 0
+// The lower bound never appears; it is inside the recorded distance.
+// CHECK: [[DNEW:%.+]] = getelementptr i8, ptr [[DVAR]], i64 {{%.+}}
+// CHECK-NEXT: store ptr [[DVAR]], ptr [[DB]],
+// CHECK-NEXT: store ptr [[DNEW]], ptr [[DP]],
+// The body's own use of i is a by-copy literal in slot 1, and is re-read.
+// CHECK-NEXT: getelementptr inbounds nuw %struct.anon{{[.0-9]*}}, ptr {{%.+}}, i32 0, i32 1
+// CHECK-NEXT: [[IREF:%.+]] = load ptr,
+// CHECK-NEXT: [[IVAL:%.+]] = load i32, ptr [[IREF]],
+// CHECK-NEXT: [[IB:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 1
+// CHECK-NEXT: [[IP:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 1
+// CHECK-NEXT: store i32 [[IVAL]], ptr [[IB]],
+// CHECK-NEXT: store i32 [[IVAL]], ptr [[IP]],
+// CHECK-NEXT: ret void
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[MEMBER:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// CHECK: define internal void [[MEMBER]](ptr noundef %0, ptr noundef %1)
+// The combined entry for the struct, then the member-of entry, both based on
+// the struct's capture.
+// CHECK: [[MVAR:%.+]] = load ptr, ptr {{%.+}}, align 8, !nonnull
+// CHECK-NEXT: [[MB:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 0
+// CHECK-NEXT: [[MP:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 0
+// CHECK: store ptr [[MVAR]], ptr [[MB]],
+// CHECK-NEXT: store ptr {{%.+}}, ptr [[MP]],
+// CHECK: [[MVAR2:%.+]] = load ptr, ptr {{%.+}}, align 8, !nonnull
+// CHECK-NEXT: [[MB2:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 1
+// CHECK-NEXT: [[MP2:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 1
+// CHECK: store ptr [[MVAR2]], ptr [[MB2]],
+// CHECK-NEXT: store ptr {{%.+}}, ptr [[MP2]],
+// CHECK-NEXT: ret void
+
+// Declined, so the runtime keeps its refuse-to-relocate behaviour.
+// CHECK: define {{.*}}@double_indirection
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr null)
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[SAVED:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// CHECK: define internal void [[SAVED]](ptr noundef %0, ptr noundef %1)
+// CHECK-NOT: getelementptr ptr,
+// CHECK: ret void
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[INNER:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// CHECK: define internal void [[INNER]](ptr noundef %0, ptr noundef %1)
+// CHECK-NOT: getelementptr ptr,
+// CHECK: ret void
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[LITERAL:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// CHECK: define internal void [[LITERAL]](ptr noundef %0, ptr noundef %1)
+// Slot 0 is A's address, relocated as in whole().
+// CHECK: getelementptr i8, ptr
+// CHECK-NEXT: store ptr {{%.+}}, ptr {{%.+}},
+// CHECK-NEXT: store ptr {{%.+}}, ptr {{%.+}},
+// n, through the taskgraph record, at its own width in both slots.
+// CHECK-NEXT: getelementptr inbounds nuw %struct.anon{{[.0-9]*}}, ptr {{%.+}}, i32 0, i32 1
+// CHECK-NEXT: [[NREF:%.+]] = load ptr,
+// CHECK-NEXT: [[NVAL:%.+]] = load i32, ptr [[NREF]],
+// CHECK-NEXT: [[NB:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 1
+// CHECK-NEXT: [[NP:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 1
+// CHECK-NEXT: store i32 [[NVAL]], ptr [[NB]],
+// CHECK-NEXT: store i32 [[NVAL]], ptr [[NP]],
+// GS, from the global.
+// CHECK-NEXT: [[GVAL:%.+]] = load i32, ptr @GS,
+// CHECK-NEXT: [[GB:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 2
+// CHECK-NEXT: [[GP:%.+]] = getelementptr ptr, ptr {{%.+}}, i64 2
+// CHECK-NEXT: store i32 [[GVAL]], ptr [[GB]],
+// CHECK-NEXT: store i32 [[GVAL]], ptr [[GP]],
+// CHECK-NEXT: ret void
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[KEPT:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// CHECK: define internal void [[KEPT]](ptr noundef %0, ptr noundef %1)
+// CHECK-NOT: getelementptr ptr,
+// CHECK: ret void
+
+// CHECK: call void @__kmpc_taskgraph_target_update({{.*}}, ptr [[STRIDED:@.omp.taskgraph.target.relocate[0-9.]*]])
+//
+// CHECK: define internal void [[STRIDED]](ptr noundef %0, ptr noundef %1, ptr noundef %2)
+// A data node passes the recorded base-pointer array first and the pointer
+// array second.
+// CHECK: [[SBASES:%.+]] = load ptr, ptr %.addr,
+// CHECK-NEXT: [[SPTRS:%.+]] = load ptr, ptr %.addr1,
+// CHECK: [[SVAR:%.+]] = load ptr, ptr {{%.+}}, align 8, !nonnull
+// CHECK-NEXT: [[SB:%.+]] = getelementptr ptr, ptr [[SBASES]], i64 0
+// CHECK-NEXT: [[SP:%.+]] = getelementptr ptr, ptr [[SPTRS]], i64 0
+// No recorded distance is read, and the slot holding the descriptor pointer is
+// left as the runtime wrote it.
+// CHECK-NEXT: store ptr [[SVAR]], ptr [[SB]],
+// CHECK-NEXT: ret void
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_target_this_relocate_codegen.cpp b/clang/test/OpenMP/taskgraph_target_this_relocate_codegen.cpp
new file mode 100644
index 00000000000000..9640aea4677763
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_target_this_relocate_codegen.cpp
@@ -0,0 +1,85 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-linux-gnu -fopenmp-targets=x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// A taskgraph in a member function. A replay starts from the taskgraph
+// construct's enclosing data environment (OpenMP 6.0 [14.3]), whose `this` is
+// the object the function was called for this time, so a recorded target
+// construct's relocation helper re-derives the base of every map entry reached
+// through `this` from the `this` the taskgraph's record holds -- by value, so
+// it is loaded from the record rather than taken as its address -- and moves
+// the begin pointer with it. That covers an explicit section of a member, the
+// entry for the object a region maps implicitly to reach its members along
+// with those members, and the same entries on a data directive.
+
+struct S {
+ int M;
+ int Out[4];
+ void section() {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : Out[0 : 4])
+ Out[0] = 1;
+ }
+ }
+ void implicit() {
+#pragma omp taskgraph
+ {
+#pragma omp target
+ Out[0] = M;
+ }
+ }
+ void data() {
+#pragma omp taskgraph
+ {
+#pragma omp target enter data map(to : Out[0 : 4])
+ }
+ }
+};
+
+void f(S &s) {
+ s.section();
+ s.implicit();
+ s.data();
+}
+
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[SECTION:@.omp.taskgraph.target.relocate[.0-9]*]])
+// Each entry is based on `this`, loaded from the record: base `this`, begin
+// moved by the recorded distance.
+// CHECK: define internal {{(noundef i32|void)}} [[SECTION]](ptr noundef %0, ptr noundef %1)
+// CHECK: [[KA:%.+]] = load ptr, ptr %.addr,
+// CHECK-NEXT: getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr [[KA]], i32 0, i32 2
+// CHECK-NEXT: [[BASES:%.+]] = load ptr,
+// CHECK-NEXT: getelementptr inbounds nuw %struct.__tgt_kernel_arguments, ptr [[KA]], i32 0, i32 3
+// CHECK-NEXT: [[PTRS:%.+]] = load ptr,
+// CHECK-NEXT: [[REC:%.+]] = load ptr, ptr %.addr1,
+// CHECK-NEXT: [[THISF:%.+]] = getelementptr inbounds nuw %struct.anon{{[.0-9]*}}, ptr [[REC]], i32 0, i32 0
+// CHECK-NEXT: [[THIS:%.+]] = load ptr, ptr [[THISF]],
+// CHECK-NEXT: [[BSLOT:%.+]] = getelementptr ptr, ptr [[BASES]], i64 0
+// CHECK-NEXT: [[PSLOT:%.+]] = getelementptr ptr, ptr [[PTRS]], i64 0
+// CHECK-NEXT: [[OLDB:%.+]] = load ptr, ptr [[BSLOT]],
+// CHECK-NEXT: [[OLDP:%.+]] = load ptr, ptr [[PSLOT]],
+// CHECK-NEXT: [[P:%.+]] = ptrtoaddr ptr [[OLDP]] to i64
+// CHECK-NEXT: [[B:%.+]] = ptrtoaddr ptr [[OLDB]] to i64
+// CHECK-NEXT: [[D:%.+]] = sub i64 [[P]], [[B]]
+// CHECK-NEXT: [[NEWP:%.+]] = getelementptr i8, ptr [[THIS]], i64 [[D]]
+// CHECK-NEXT: store ptr [[THIS]], ptr [[BSLOT]],
+// CHECK-NEXT: store ptr [[NEWP]], ptr [[PSLOT]],
+
+// The entry for the object and its two members.
+// CHECK: call i32 @__kmpc_taskgraph_target({{.*}}, ptr [[IMPLICIT:@.omp.taskgraph.target.relocate[.0-9]*]])
+// CHECK: define internal {{(noundef i32|void)}} [[IMPLICIT]](ptr noundef %0, ptr noundef %1)
+// CHECK-COUNT-3: getelementptr inbounds nuw %struct.anon{{[.0-9]*}}, ptr %{{[0-9]+}}, i32 0, i32 0
+// CHECK-NOT: getelementptr inbounds nuw %struct.anon
+// CHECK: ret {{i32 0|void}}
+
+// The entry for the object and the member, on the data directive.
+// CHECK: call void @__kmpc_taskgraph_target_enter_data({{.*}}, ptr [[DATA:@.omp.taskgraph.target.relocate[.0-9]*]])
+// CHECK: define internal {{(noundef i32|void)}} [[DATA]](ptr noundef %0, ptr noundef %1, ptr noundef %2)
+// CHECK-COUNT-2: getelementptr inbounds nuw %struct.anon{{[.0-9]*}}, ptr %{{[0-9]+}}, i32 0, i32 0
+// CHECK-NOT: getelementptr inbounds nuw %struct.anon
+// CHECK: ret {{i32 0|void}}
+
+#endif
diff --git a/offload/test/offloading/taskgraph_target_this.cpp b/offload/test/offloading/taskgraph_target_this.cpp
new file mode 100644
index 00000000000000..3070769abd7db9
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_this.cpp
@@ -0,0 +1,65 @@
+// RUN: %libomptarget-compilexx-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic
+// RUN: %libomptarget-compilexx-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A taskgraph in a member function, replayed for different objects. A replay
+// starts from the taskgraph construct's enclosing data environment (OpenMP 6.0
+// [14.3]), where `this` is the object the function was called for this time,
+// and a list item named through it -- a member, or a section of one -- is
+// reconstructed from it. So every replay's maps reach this call's object:
+// an explicit section of a member array, and the object a target region maps
+// implicitly to reach its members. The same holds for a member mapped by
+// `target enter data` and unmapped by `target exit data`.
+
+#include <cstdio>
+
+struct S {
+ int M = 0;
+ int Out[4] = {0, 0, 0, 0};
+ int D[4] = {0, 0, 0, 0};
+
+ __attribute__((noinline)) void run() {
+#pragma omp taskgraph graph_id(12)
+ {
+#pragma omp target map(tofrom : Out[0 : 4])
+ Out[0] += 1;
+#pragma omp target
+ Out[1] += M;
+ }
+ }
+
+ __attribute__((noinline)) void runData() {
+#pragma omp taskgraph graph_id(13)
+ {
+#pragma omp target enter data map(to : D[0 : 4])
+#pragma omp target map(present, alloc : D[0 : 4])
+ D[0] += 1;
+#pragma omp target exit data map(from : D[0 : 4])
+ }
+ }
+};
+
+int main() {
+ S A, B;
+ for (int R = 0; R < 4; ++R) {
+ S &Obj = (R & 1) ? B : A;
+ Obj.M = 10 * R + 1;
+ Obj.run();
+ // Set on the host between runs, so each run has to copy it in.
+ Obj.D[0] = 1000 * R;
+ Obj.runData();
+ }
+ std::printf("A %d %d B %d %d\n", A.Out[0], A.Out[1], B.Out[0], B.Out[1]);
+ std::printf("D A %d B %d\n", A.D[0], B.D[0]);
+ return 0;
+}
+
+// Two runs each; A's M was 1 and 21, B's 11 and 31.
+// CHECK: A 2 22 B 2 42
+// A's last run was R == 2, B's R == 3.
+// CHECK: D A 2001 B 3001
>From ec0141cbb740702ae51c49f7b8a456a84184c78a Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 19 Aug 2026 10:21:04 -0500
Subject: [PATCH 18/29] [Offload] Transmission of taskgraphs,
libomp->libomptarget: omptarget side
When a taskgraph record contains target regions, we want to be able to
schedule those efficiently to run on offload hardware. At the moment
we have a processed taskgraph representation in libomp, but we need a
way to get that into libomptarget instead, since that is the part of
the runtime which currently handles target offload.
libomp stays in charge of replaying the taskgraph. Each maximal subtree
of its processed region tree that holds only target constructs is
transmitted to libomptarget as a subgraph of its own, and a replay that
reaches the subgraph has libomptarget run it and waits for it to
complete. Host tasks are never part of a subgraph, so libomptarget never
has to call back into libomp to run them, and a subgraph can be issued in
full without anything on the host having to run before it finishes.
The representation here is somewhat decoupled from the one used within
libomp, partly because of the different coding style adopted by each
library, and partly because different aspects of the representation are
important when recording/host-replaying the graph vs. executing it on an
offload device. Particular care has been taken over avoiding the need
for strict binary compatibility (e.g. structure layout) between libomp
and libomptarget -- the interface between the two is via function calls
with POD argument types.
The transferred kernel args and mapping arrays are shallow copied/
cross-linked to the recorded taskgraph data in libomp. As such, the
libomp-side data must outlive the libomptarget-side data: that is part
of the contract between the two libraries.
We use a two-pass method to transmit the graph-shaped data so that we
can use a single allocation block on the libomptarget side (partly for
efficiency, but also so pointers don't need to be moved mid-stream).
Once received, the subgraph is offered to the current plugin, which has
the opportunity to finalize it into some API-specific executable form,
or decline it -- in which case libomp issues the subgraph's target
constructs itself, one at a time (which always works, but may be less
optimal on some given configuration).
A replay makes two calls per subgraph: __tgt_taskgraph_begin_replay when
the replay starts, once libomp has brought the recorded arguments up to
date, and __tgt_taskgraph_replay when the replay reaches the subgraph,
which runs it to completion. A third subgraph-specific call is
__tgt_taskgraph_destroy, called when the record is freed during a reset.
The structures in this patch handle the coarse control portions of a
taskgraph, i.e. what runs, and when. Where target constructs find their
data is handled by a later patch.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/225708
---
offload/include/Shared/TaskGraph.h | 328 ++++++++++++++++++
offload/include/Shared/TaskGraphBuilder.h | 227 ++++++++++++
offload/include/omptarget.h | 108 ++++++
offload/libomptarget/exports | 19 +
offload/libomptarget/interface.cpp | 322 ++++++++++++++++-
offload/libomptarget/omptarget.cpp | 18 +
offload/libomptarget/private.h | 6 +
.../common/include/PluginInterface.h | 41 +++
.../common/src/PluginInterface.cpp | 43 +++
9 files changed, 1099 insertions(+), 13 deletions(-)
create mode 100644 offload/include/Shared/TaskGraph.h
create mode 100644 offload/include/Shared/TaskGraphBuilder.h
diff --git a/offload/include/Shared/TaskGraph.h b/offload/include/Shared/TaskGraph.h
new file mode 100644
index 00000000000000..fef340bb1dd879
--- /dev/null
+++ b/offload/include/Shared/TaskGraph.h
@@ -0,0 +1,328 @@
+//===-- Shared/TaskGraph.h - Processed-taskgraph object model ----- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The representation of a target subgraph of a processed OpenMP taskgraph --
+// a subtree of its regions that holds target constructs and nothing else --
+// shared between libomptarget and the device plugins. libomp runs the rest of
+// the taskgraph itself, and has libomptarget run each subgraph a plugin claims
+// when the replay reaches it.
+//
+// The representation here is somewhat decoupled from the one used within
+// libomp, partly because of the different code styles adopted by each library,
+// and partly because different aspects of the representation are important when
+// recording/host-replaying the graph vs. executing it on an offload device.
+// Particular care has been taken over avoiding the need for strict binary
+// compatibility (e.g. structure layout) between libomp and libomptarget -- the
+// interface between the two is via function calls with POD argument types.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_SHARED_TASKGRAPH_H
+#define OMPTARGET_SHARED_TASKGRAPH_H
+
+#include "omptarget.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Casting.h"
+
+#include <cassert>
+#include <cstdint>
+#include <cstdlib>
+#include <new>
+#include <vector>
+
+namespace llvm {
+namespace omp {
+namespace target {
+
+//===----------------------------------------------------------------------===//
+// Object model (LLVM-style RTTI)
+//
+// libomp streams each subgraph in preorder, with each region announcing
+// its child count up front (prefix notation). We materialize it as a tree of
+// the types below, laid out in a single contiguous block: see BuildCursor and
+// the two-pass allocation it drives.
+//
+//===----------------------------------------------------------------------===//
+
+/// Taskgraph node/region type discriminator. Order is significant here.
+enum class TGKind : uint8_t {
+ // Interior regions (own an ordered child list).
+ ParRegion,
+ SeqRegion,
+ ExclRegion,
+ IrreducibleRegion,
+
+ // Leaves.
+ TargetNode,
+ // Data leaves form their own sub-range at the tail of the leaf range.
+ TargetEnterDataNode,
+ TargetExitDataNode,
+ TargetUpdateDataNode,
+};
+
+struct TaskGraphRegion;
+
+/// Common base for every taskgraph element.
+struct TaskGraphElement {
+ const TGKind Kind;
+ TaskGraphRegion *Parent = nullptr; // null iff this is the graph root
+ uint32_t ChildIndex = 0; // ordinal within Parent's child list
+
+ TGKind getKind() const { return Kind; }
+
+protected:
+ explicit TaskGraphElement(TGKind K) : Kind(K) {}
+};
+
+//===----------------------------------------------------------------------===//
+// Regions (interior nodes)
+//===----------------------------------------------------------------------===//
+
+/// An interior region owning an ordered list of children. The child pointer
+/// array is bump-allocated from the same block and filled in child order by
+/// the build cursor; its length is known up front from the stream's prefix
+/// count, so the region is fully wired before its children are streamed in.
+struct TaskGraphRegion : TaskGraphElement {
+ uint32_t NumChildren = 0;
+ TaskGraphElement **Children = nullptr;
+
+ llvm::MutableArrayRef<TaskGraphElement *> children() {
+ return {Children, NumChildren};
+ }
+ llvm::ArrayRef<const TaskGraphElement *> children() const {
+ return llvm::ArrayRef<TaskGraphElement *>(Children, NumChildren);
+ }
+
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() >= TGKind::ParRegion &&
+ E->getKind() <= TGKind::IrreducibleRegion;
+ }
+
+protected:
+ TaskGraphRegion(TGKind K, uint32_t N) : TaskGraphElement(K), NumChildren(N) {}
+};
+
+/// Parallel region (children may run concurrently).
+struct TaskGraphParRegion : TaskGraphRegion {
+ explicit TaskGraphParRegion(uint32_t N)
+ : TaskGraphRegion(TGKind::ParRegion, N) {}
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() == TGKind::ParRegion;
+ }
+};
+
+/// Sequential region (children run in order).
+struct TaskGraphSeqRegion : TaskGraphRegion {
+ explicit TaskGraphSeqRegion(uint32_t N)
+ : TaskGraphRegion(TGKind::SeqRegion, N) {}
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() == TGKind::SeqRegion;
+ }
+};
+
+/// Children are mutually exclusive (mutexinoutset), unordered.
+struct TaskGraphExclRegion : TaskGraphRegion {
+ explicit TaskGraphExclRegion(uint32_t N)
+ : TaskGraphRegion(TGKind::ExclRegion, N) {}
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() == TGKind::ExclRegion;
+ }
+};
+
+/// An intra-region dependence edge of an irreducible region, naming children by
+/// ordinal. Both endpoints name a child: the region's sources and sinks are
+/// whichever children have no incoming or no outgoing edge, same as for the
+/// corresponding edges in libomp.
+struct TaskGraphEdge {
+ int32_t Src;
+ int32_t Dst;
+};
+
+/// An irreducible region. Dependencies between children are represented by
+/// an explicit edge array (a pointer into the same allocation block as the
+/// rest of the transmitted graph).
+struct TaskGraphIrreducibleRegion : TaskGraphRegion {
+ uint32_t NumEdges = 0;
+ uint32_t EdgeFill = 0; // transient build cursor into Edges
+ TaskGraphEdge *Edges = nullptr; // len == NumEdges, into same block
+
+ llvm::ArrayRef<TaskGraphEdge> edges() const { return {Edges, NumEdges}; }
+
+ explicit TaskGraphIrreducibleRegion(uint32_t N)
+ : TaskGraphRegion(TGKind::IrreducibleRegion, N) {}
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() == TGKind::IrreducibleRegion;
+ }
+};
+
+//===----------------------------------------------------------------------===//
+// Leaves (nodes)
+//===----------------------------------------------------------------------===//
+
+/// Common base for the leaf nodes.
+struct TaskGraphNode : TaskGraphElement {
+ int64_t DeviceId = OFFLOAD_DEVICE_DEFAULT;
+
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() >= TGKind::TargetNode &&
+ E->getKind() <= TGKind::TargetUpdateDataNode;
+ }
+
+protected:
+ explicit TaskGraphNode(TGKind K) : TaskGraphElement(K) {}
+};
+
+/// A target kernel launch. KernelArgs is an opaque KernelArgsTy*, deep-copied
+/// by libomp at record time.
+struct TaskGraphTargetNode : TaskGraphNode {
+ int32_t NumTeams = 0;
+ int32_t ThreadLimit = 0;
+ void *HostPtr = nullptr;
+ void *KernelArgs = nullptr;
+
+ TaskGraphTargetNode() : TaskGraphNode(TGKind::TargetNode) {}
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() == TGKind::TargetNode;
+ }
+};
+
+/// Base for target enter/exit/update data ops. The specific op is encoded by
+/// the Kind. The arg arrays are libomp-owned (no copy).
+struct TaskGraphTargetDataNode : TaskGraphNode {
+ int32_t ArgNum = 0;
+ void **ArgsBase = nullptr;
+ void **Args = nullptr;
+ int64_t *ArgSizes = nullptr;
+ int64_t *ArgTypes = nullptr;
+ void **ArgNames = nullptr;
+ void **ArgMappers = nullptr;
+
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() >= TGKind::TargetEnterDataNode &&
+ E->getKind() <= TGKind::TargetUpdateDataNode;
+ }
+
+protected:
+ explicit TaskGraphTargetDataNode(TGKind K) : TaskGraphNode(K) {}
+};
+
+struct TaskGraphTargetEnterDataNode : TaskGraphTargetDataNode {
+ TaskGraphTargetEnterDataNode()
+ : TaskGraphTargetDataNode(TGKind::TargetEnterDataNode) {}
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() == TGKind::TargetEnterDataNode;
+ }
+};
+
+struct TaskGraphTargetExitDataNode : TaskGraphTargetDataNode {
+ TaskGraphTargetExitDataNode()
+ : TaskGraphTargetDataNode(TGKind::TargetExitDataNode) {}
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() == TGKind::TargetExitDataNode;
+ }
+};
+
+struct TaskGraphTargetUpdateDataNode : TaskGraphTargetDataNode {
+ TaskGraphTargetUpdateDataNode()
+ : TaskGraphTargetDataNode(TGKind::TargetUpdateDataNode) {}
+ static bool classof(const TaskGraphElement *E) {
+ return E->getKind() == TGKind::TargetUpdateDataNode;
+ }
+};
+
+//===----------------------------------------------------------------------===//
+// libomptarget services available to a plugin while lowering a graph
+//===----------------------------------------------------------------------===//
+
+/// What a plugin needs from libomptarget to lower a recorded graph, passed as a
+/// function pointer rather than called directly.
+///
+/// A plugin cannot name a libomptarget symbol: plugins are static archives
+/// linked into both libomptarget.so and LLVMOffload, so a direct call would
+/// leave LLVMOffload with an undefined reference.
+///
+/// Every DeviceId passed here is a concrete device: libomptarget normalizes
+/// OFFLOAD_DEVICE_DEFAULT away across the whole tree before a plugin sees it.
+struct TaskGraphServicesTy {
+ /// Resolve a recorded host function pointer to the plugin-side kernel object
+ /// (a GenericKernelTy *), or null if the device has no matching entry.
+ void *(*resolveKernel)(int64_t DeviceId, void *HostPtr) = nullptr;
+};
+
+//===----------------------------------------------------------------------===//
+// The graph
+//===----------------------------------------------------------------------===//
+
+class TaskGraphBuilder;
+
+/// Where the build pass links the next streamed element.
+struct BuildCursor {
+ /// The region being filled, or null at the root.
+ TaskGraphRegion *Cur = nullptr;
+ /// The next slot in Cur->children().
+ uint32_t Index = 0;
+};
+
+/// A processed taskgraph, streamed in by libomp and built by TaskGraphBuilder
+/// (Shared/TaskGraphBuilder.h). A plugin is handed a const graph, which it
+/// only reads.
+class TaskGraphTy {
+public:
+ TaskGraphTy(int64_t DeviceId, uintptr_t GraphId)
+ : DeviceId(DeviceId), GraphId(GraphId) {}
+ ~TaskGraphTy() { std::free(Block); }
+ TaskGraphTy(const TaskGraphTy &) = delete;
+ TaskGraphTy &operator=(const TaskGraphTy &) = delete;
+
+ /// The device the graph is replayed on, concrete once the graph is offered
+ /// to a plugin.
+ int64_t DeviceId;
+ uintptr_t GraphId;
+ /// The claiming plugin's own form of the graph: whatever its finalize
+ /// returned, opaque to everything but that plugin. Null for a graph no
+ /// plugin claimed.
+ void *PluginGraph = nullptr;
+
+ /// Filled in by libomptarget before the graph is offered to a plugin.
+ TaskGraphServicesTy Services;
+
+ /// The top of the tree: the first element streamed outside any region. A
+ /// region in general, but a graph holding a single leaf arrives as that leaf.
+ TaskGraphElement *getRoot() { return Root; }
+ const TaskGraphElement *getRoot() const { return Root; }
+
+ /// The leaves in preorder. Empty for a graph that was not well formed.
+ ArrayRef<TaskGraphNode *> leaves() { return Leaves; }
+ ArrayRef<const TaskGraphNode *> leaves() const {
+ return ArrayRef<TaskGraphNode *>(Leaves);
+ }
+
+private:
+ friend class TaskGraphBuilder;
+
+ TaskGraphElement *Root = nullptr;
+ std::vector<TaskGraphNode *> Leaves;
+
+ /// The single block holding the whole tree, null on the measuring pass.
+ char *Block = nullptr;
+ /// The block's size: accumulated on the measuring pass, given on the build
+ /// pass.
+ size_t MeasuredBytes = 0;
+ /// How much of the block the build pass has used.
+ size_t BlockUsed = 0;
+ BuildCursor Cursor;
+ /// How the stream broke the transmission protocol, if it did.
+ const char *Malformed = nullptr;
+};
+
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OMPTARGET_SHARED_TASKGRAPH_H
diff --git a/offload/include/Shared/TaskGraphBuilder.h b/offload/include/Shared/TaskGraphBuilder.h
new file mode 100644
index 00000000000000..1277877ca274ea
--- /dev/null
+++ b/offload/include/Shared/TaskGraphBuilder.h
@@ -0,0 +1,227 @@
+//===-- Shared/TaskGraphBuilder.h - Build a streamed taskgraph ---- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Construction of a TaskGraphTy from the preorder stream libomp sends through
+// the taskgraph transmission ABI (omptarget.h). Used by libomptarget, and by
+// the unit tests that stream graphs the same way; a plugin only ever reads a
+// finished graph.
+//
+// The stream is sent twice. On the measuring pass the builder only adds up
+// the bytes the tree will occupy; on the build pass it lays the tree out in a
+// single block of that size, which never moves while it is filled.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_SHARED_TASKGRAPHBUILDER_H
+#define OMPTARGET_SHARED_TASKGRAPHBUILDER_H
+
+#include "Shared/TaskGraph.h"
+
+#include <cassert>
+#include <cstddef>
+#include <cstdlib>
+#include <new>
+#include <vector>
+
+namespace llvm {
+namespace omp {
+namespace target {
+
+class TaskGraphBuilder {
+public:
+ explicit TaskGraphBuilder(TaskGraphTy &G) : G(G) {}
+
+ /// Begin a pass over the stream: the measuring pass for a \p ByteSize of
+ /// zero, and otherwise the build pass into a block of \p ByteSize bytes.
+ static TaskGraphTy *start(int64_t DeviceId, uintptr_t GraphId,
+ size_t ByteSize) {
+ auto *G = new TaskGraphTy(DeviceId, GraphId);
+ G->MeasuredBytes = ByteSize;
+ if (ByteSize)
+ G->Block = static_cast<char *>(std::malloc(ByteSize));
+ return G;
+ }
+
+ /// End the pass, returning the size of block the graph needs. On the build
+ /// pass, a well-formed tree also gets its leaves collected.
+ size_t finish() {
+ if (!building())
+ return alignUp(G.MeasuredBytes, alignof(std::max_align_t));
+ assert(G.BlockUsed <= G.MeasuredBytes && "taskgraph block overrun");
+ if (!whyMalformed(G))
+ collectLeaves(G.Root, G.Leaves);
+ return G.MeasuredBytes;
+ }
+
+ /// Whether this is the build pass rather than the measuring pass.
+ bool building() const { return G.Block != nullptr; }
+
+ /// Account for (measuring pass) or bump-allocate (build pass) \p Size bytes.
+ void *alloc(size_t Size, size_t Align) {
+ if (!building()) {
+ G.MeasuredBytes = alignUp(G.MeasuredBytes, Align) + Size;
+ return nullptr;
+ }
+ G.BlockUsed = alignUp(G.BlockUsed, Align);
+ assert(G.BlockUsed + Size <= G.MeasuredBytes &&
+ "taskgraph single-block overflow");
+ void *P = G.Block + G.BlockUsed;
+ G.BlockUsed += Size;
+ return P;
+ }
+
+ /// Allocate a leaf, constructed on the build pass. Null on the measuring
+ /// pass.
+ template <class NodeT> NodeT *makeLeaf() {
+ void *P = alloc(sizeof(NodeT), alignof(NodeT));
+ return building() ? new (P) NodeT() : nullptr;
+ }
+
+ /// Allocate a region and its child-pointer array, constructed on the build
+ /// pass. Null on the measuring pass.
+ template <class RegionT> RegionT *makeRegion(int32_t NumChildren) {
+ void *P = alloc(sizeof(RegionT), alignof(RegionT));
+ void *Children = alloc(sizeof(TaskGraphElement *) * NumChildren,
+ alignof(TaskGraphElement *));
+ if (!building())
+ return nullptr;
+ auto *R = new (P) RegionT(static_cast<uint32_t>(NumChildren));
+ R->Children = static_cast<TaskGraphElement **>(Children);
+ return R;
+ }
+
+ /// Allocate an irreducible region with room for \p NumEdges edges.
+ TaskGraphIrreducibleRegion *makeIrreducibleRegion(int32_t NumChildren,
+ int32_t NumEdges) {
+ auto *R = makeRegion<TaskGraphIrreducibleRegion>(NumChildren);
+ void *Edges =
+ alloc(sizeof(TaskGraphEdge) * NumEdges, alignof(TaskGraphEdge));
+ if (!building())
+ return nullptr;
+ R->NumEdges = static_cast<uint32_t>(NumEdges);
+ R->Edges = static_cast<TaskGraphEdge *>(Edges);
+ return R;
+ }
+
+ // Build-cursor operations, for the build pass only.
+
+ /// Link \p E as the next child of the region being filled, or as the root.
+ void linkChild(TaskGraphElement *E) {
+ BuildCursor &Cursor = G.Cursor;
+ E->Parent = Cursor.Cur;
+ E->ChildIndex = Cursor.Index;
+ if (Cursor.Cur) {
+ if (Cursor.Index == Cursor.Cur->NumChildren)
+ return setMalformed("more children than their region announced");
+ Cursor.Cur->Children[Cursor.Index++] = E;
+ return;
+ }
+ if (G.Root)
+ return setMalformed("more than one root");
+ G.Root = E;
+ }
+ /// Link \p R and fill it with the children that follow.
+ void pushRegion(TaskGraphRegion *R) {
+ linkChild(R);
+ G.Cursor.Cur = R;
+ G.Cursor.Index = 0;
+ }
+ /// Finish the region being filled.
+ void popRegion() {
+ BuildCursor &Cursor = G.Cursor;
+ if (!Cursor.Cur)
+ return setMalformed("a region end with no region open");
+ // Resume the parent at the slot after the region just finished.
+ uint32_t Slot = Cursor.Cur->ChildIndex;
+ Cursor.Cur = Cursor.Cur->Parent;
+ Cursor.Index = Slot + 1;
+ }
+ /// Add an edge to the irreducible region being filled.
+ void addEdge(int32_t Src, int32_t Dst) {
+ auto *Irr =
+ llvm::dyn_cast_if_present<TaskGraphIrreducibleRegion>(G.Cursor.Cur);
+ if (!Irr)
+ return setMalformed("an edge outside an irreducible region");
+ if (Irr->EdgeFill == Irr->NumEdges)
+ return setMalformed("more edges than their irreducible region announced");
+ if (Src < 0 || Dst < 0 || static_cast<uint32_t>(Src) >= Irr->NumChildren ||
+ static_cast<uint32_t>(Dst) >= Irr->NumChildren)
+ return setMalformed("an edge naming no child of its region");
+ Irr->Edges[Irr->EdgeFill++] = {Src, Dst};
+ }
+
+ /// An explanation for why \p G is malformed if is it, else null.
+ static const char *whyMalformed(const TaskGraphTy &G) {
+ if (G.Malformed)
+ return G.Malformed;
+ if (!G.Root)
+ return "no root";
+ return whyMalformed(G.Root);
+ }
+
+private:
+ /// Note the first way the stream broke the transmission protocol.
+ void setMalformed(const char *Why) {
+ if (!G.Malformed)
+ G.Malformed = Why;
+ }
+
+ static size_t alignUp(size_t X, size_t A) { return (X + A - 1) & ~(A - 1); }
+
+ static const char *whyMalformed(const TaskGraphElement *E) {
+ const auto *R = llvm::dyn_cast<TaskGraphRegion>(E);
+ if (!R)
+ return nullptr;
+ for (const TaskGraphElement *C : R->children()) {
+ if (!C)
+ return "fewer children than their region announced";
+ if (const char *Why = whyMalformed(C))
+ return Why;
+ }
+ const auto *Irr = llvm::dyn_cast<TaskGraphIrreducibleRegion>(R);
+ if (!Irr)
+ return nullptr;
+ if (Irr->EdgeFill != Irr->NumEdges)
+ return "fewer edges than their irreducible region announced";
+
+ // Kahn's algorithm: the region is acyclic iff every child gets ordered.
+ std::vector<uint32_t> InDegree(Irr->NumChildren, 0);
+ for (const TaskGraphEdge &Edge : Irr->edges())
+ ++InDegree[Edge.Dst];
+ std::vector<uint32_t> Ready;
+ for (uint32_t I = 0; I < Irr->NumChildren; ++I)
+ if (!InDegree[I])
+ Ready.push_back(I);
+ for (size_t Head = 0; Head < Ready.size(); ++Head)
+ for (const TaskGraphEdge &Edge : Irr->edges())
+ if (static_cast<uint32_t>(Edge.Src) == Ready[Head] &&
+ !--InDegree[Edge.Dst])
+ Ready.push_back(Edge.Dst);
+ return Ready.size() == Irr->NumChildren
+ ? nullptr
+ : "a cycle among an irreducible region's edges";
+ }
+
+ static void collectLeaves(TaskGraphElement *E,
+ std::vector<TaskGraphNode *> &Out) {
+ if (auto *R = llvm::dyn_cast<TaskGraphRegion>(E)) {
+ for (TaskGraphElement *C : R->children())
+ collectLeaves(C, Out);
+ return;
+ }
+ Out.push_back(llvm::cast<TaskGraphNode>(E));
+ }
+
+ TaskGraphTy &G;
+};
+
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OMPTARGET_SHARED_TASKGRAPHBUILDER_H
diff --git a/offload/include/omptarget.h b/offload/include/omptarget.h
index f20670b7941b9b..cb98f16c85086e 100644
--- a/offload/include/omptarget.h
+++ b/offload/include/omptarget.h
@@ -111,6 +111,10 @@ inline bool isPrivate(int64_t MapType) {
inline bool isLiteral(int64_t MapType) {
return MapType & OMP_TGT_MAPTYPE_LITERAL;
}
+/// Non-contiguous descriptor (used by non-contiguous target update).
+inline bool isNonContig(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_NON_CONTIG;
+}
/// Firstprivate aggregate whose clause carried the "saved" modifier.
inline bool isSaved(int64_t MapType) { return MapType & OMP_TGT_MAPTYPE_SAVED; }
} // namespace maptype
@@ -487,6 +491,110 @@ void *__tgt_get_mapped_ptr(int64_t DeviceId, const void *HostPtr);
// code for details.
void __tgt_register_rpc_callback(unsigned (*Callback)(void *, unsigned));
+//===----------------------------------------------------------------------===//
+// Taskgraph transmission ABI (libomp -> libomptarget)
+//
+// libomp runs a replayed taskgraph itself. Each maximal subtree of its
+// processed region tree that holds only target constructs is a *subgraph*,
+// which libomp streams to libomptarget through the preorder calls below, once,
+// when the taskgraph is recorded. A replay that reaches the subgraph has
+// libomptarget run it on the device, and waits for it. See Shared/TaskGraph.h
+// for the resulting data structures within this library.
+//
+// Everything a subgraph names is libomp's: the recorded kernel arguments and
+// map arrays, which libomp brings up to date for each replay before any
+// subgraph runs. Only plain values and opaque pointers cross this interface,
+// so no structure layout has to be kept in step between the two libraries.
+//===----------------------------------------------------------------------===//
+
+/// Begin streaming a subgraph. The first element streamed after this, a
+/// region or a single leaf, is the subgraph's root.
+///
+/// \p ByteSize drives the two-pass, single-block construction. On the first
+/// (measuring) pass libomp passes 0; libomptarget only accumulates the byte
+/// size the subgraph will occupy and returns it from __tgt_taskgraph_end.
+/// libomp then replays the identical stream on a second pass, passing that
+/// size back here so libomptarget can allocate the whole subgraph in one
+/// exactly-sized block that never moves during construction.
+void *__tgt_taskgraph_start(int64_t DeviceId, uintptr_t GraphId,
+ size_t ByteSize);
+
+/// Preorder region bracketing. \p NodeCount is the region's immediate-child
+/// count.
+void __tgt_taskgraph_start_parallel(void *Graph, int32_t NodeCount);
+void __tgt_taskgraph_end_parallel(void *Graph);
+void __tgt_taskgraph_start_sequential(void *Graph, int32_t NodeCount);
+void __tgt_taskgraph_end_sequential(void *Graph);
+void __tgt_taskgraph_start_exclusive(void *Graph, int32_t NodeCount);
+void __tgt_taskgraph_end_exclusive(void *Graph);
+
+/// Irreducible region bracketing. Child nodes are transmitted first, followed
+/// by edges.
+/// \p NodeCount is the immediate-child count (as for the other containers) and
+/// \p EdgeCount is the number of edges.
+void __tgt_taskgraph_start_irreducible(void *Graph, int32_t NodeCount,
+ int32_t EdgeCount);
+void __tgt_taskgraph_end_irreducible(void *Graph);
+
+/// A single edge of an irreducible region. \p SrcChild / \p DstChild are
+/// 0-based child indices within that region, and the edges must not form a
+/// cycle. A subgraph breaking either rule is not offloaded.
+void __tgt_taskgraph_emit_edge(void *Graph, int32_t SrcChild, int32_t DstChild);
+
+/// Emit a target kernel leaf. \p KernelArgs is libomp's deep copy of the
+/// kernel-arguments blob (opaque KernelArgsTy*).
+void __tgt_taskgraph_emit_target(void *Graph, int64_t DeviceId,
+ int32_t NumTeams, int32_t ThreadLimit,
+ void *HostPtr, void *KernelArgs);
+
+/// Emit a target enter-data / exit-data / update leaf. The map arrays are
+/// libomp-owned deep copies.
+void __tgt_taskgraph_emit_target_enter_data(void *Graph, int64_t DeviceId,
+ int32_t ArgNum, void **ArgsBase,
+ void **Args, int64_t *ArgSizes,
+ int64_t *ArgTypes, void **ArgNames,
+ void **ArgMappers);
+void __tgt_taskgraph_emit_target_exit_data(void *Graph, int64_t DeviceId,
+ int32_t ArgNum, void **ArgsBase,
+ void **Args, int64_t *ArgSizes,
+ int64_t *ArgTypes, void **ArgNames,
+ void **ArgMappers);
+void __tgt_taskgraph_emit_target_update(void *Graph, int64_t DeviceId,
+ int32_t ArgNum, void **ArgsBase,
+ void **Args, int64_t *ArgSizes,
+ int64_t *ArgTypes, void **ArgNames,
+ void **ArgMappers);
+
+/// Finish transmitting a subgraph. Returns the number of bytes the subgraph
+/// occupies (the single-block size): meaningful on the first (measuring) pass,
+/// where its return value is fed back to __tgt_taskgraph_start as
+/// \p ByteSize.
+size_t __tgt_taskgraph_end(void *Graph);
+
+/// Build the executable form of a subgraph (delegates to the active plugin).
+/// Returns OFFLOAD_SUCCESS iff a plugin claimed the subgraph and will execute
+/// it. On OFFLOAD_FAIL, libomp runs the subgraph's target constructs itself,
+/// one at a time. The subgraph is finalized for the device it was started on.
+int __tgt_taskgraph_finalize(void *Graph);
+
+/// Begin a replay of the taskgraph a finalized subgraph belongs to. Called
+/// for every subgraph when the replay starts, after libomp has brought the
+/// recorded arguments up to date and before anything in the replay runs:
+/// this is when the subgraph takes the values of its firstprivates (OpenMP
+/// 6.0 [14.3]).
+void __tgt_taskgraph_begin_replay(void *Graph);
+
+/// Run a finalized subgraph on the device and wait for it to complete:
+/// resolve its data against the current mapping state, issue it, and wait.
+/// Returns OFFLOAD_SUCCESS once it has run, or OFFLOAD_FAIL if it was declined
+/// before anything was issued, in which case libomp runs its target constructs
+/// itself. A failure after anything was issued is fatal.
+int __tgt_taskgraph_replay(void *Graph);
+
+/// Destroy a finalized taskgraph. Must be called before libomp frees the
+/// captured node payloads referenced by the graph.
+void __tgt_taskgraph_destroy(void *Graph);
+
/// Duplicate kernel args at capture time. Uses a two-pass measure/allocate
/// strategy. Used so libomp doesn't have to understand kernel arg layout.
void __tgt_taskgraph_dup_kernel_args(void *Dst, void *Src, size_t *AllocSize);
diff --git a/offload/libomptarget/exports b/offload/libomptarget/exports
index 037207d67da554..5e086c86b7a015 100644
--- a/offload/libomptarget/exports
+++ b/offload/libomptarget/exports
@@ -31,6 +31,25 @@ VERS1.0 {
__tgt_target_kernel_nowait;
__tgt_target_nowait_query;
__tgt_target_kernel_replay;
+ __tgt_taskgraph_start;
+ __tgt_taskgraph_start_parallel;
+ __tgt_taskgraph_end_parallel;
+ __tgt_taskgraph_start_sequential;
+ __tgt_taskgraph_end_sequential;
+ __tgt_taskgraph_start_exclusive;
+ __tgt_taskgraph_end_exclusive;
+ __tgt_taskgraph_start_irreducible;
+ __tgt_taskgraph_end_irreducible;
+ __tgt_taskgraph_emit_edge;
+ __tgt_taskgraph_emit_target;
+ __tgt_taskgraph_emit_target_enter_data;
+ __tgt_taskgraph_emit_target_exit_data;
+ __tgt_taskgraph_emit_target_update;
+ __tgt_taskgraph_end;
+ __tgt_taskgraph_finalize;
+ __tgt_taskgraph_begin_replay;
+ __tgt_taskgraph_replay;
+ __tgt_taskgraph_destroy;
__tgt_taskgraph_dup_kernel_args;
__tgt_taskgraph_dup_data_args;
__tgt_activate_record_replay;
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index 4beaf4512c7764..4b65077f112400 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -13,26 +13,35 @@
#include "OpenMP/OMPT/Interface.h"
#include "OffloadPolicy.h"
+#include "OpenMP/Mapping.h"
#include "OpenMP/OMPT/Callback.h"
#include "OpenMP/OffloadRTL.h"
#include "OpenMP/omp.h"
+#include "PluginManager.h"
+#include "device.h"
#include "omptarget.h"
#include "private.h"
#include "Shared/EnvironmentVar.h"
#include "Shared/Profile.h"
+#include "Shared/TaskGraph.h"
+#include "Shared/TaskGraphBuilder.h"
#include "Utils/ExponentialBackoff.h"
#include "llvm/Frontend/OpenMP/OMPConstants.h"
+#include "llvm/Support/Casting.h"
#include "llvm/Support/MathExtras.h"
+#include <atomic>
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <memory>
+#include <mutex>
+#include <new>
#include <vector>
#ifdef OMPT_SUPPORT
@@ -291,6 +300,20 @@ struct UpgradedArgBuffersTy {
llvm::SmallVector<void *, 0> Mappers;
};
+/// Give the unused dimensions of \p KernelArgs' launch a size of one.
+// FIXME: This is a WA to "calibrate" the bad work done in the front end.
+// Delete this ugly code after the front end emits proper values.
+static void correctMultiDim(KernelArgsTy &KernelArgs) {
+ auto Correct = [](uint32_t (&Val)[3]) {
+ if (Val[1] == 0)
+ Val[1] = 1;
+ if (Val[2] == 0)
+ Val[2] = 1;
+ };
+ Correct(KernelArgs.UserThreadLimit);
+ Correct(KernelArgs.UserNumBlocks);
+}
+
static KernelArgsTy *upgradeKernelArgs(KernelArgsTy *KernelArgs,
KernelArgsTy &LocalKernelArgs,
UpgradedArgBuffersTy &Bufs,
@@ -322,16 +345,7 @@ static KernelArgsTy *upgradeKernelArgs(KernelArgsTy *KernelArgs,
return &LocalKernelArgs;
}
- // FIXME: This is a WA to "calibrate" the bad work done in the front end.
- // Delete this ugly code after the front end emits proper values.
- auto CorrectMultiDim = [](uint32_t (&Val)[3]) {
- if (Val[1] == 0)
- Val[1] = 1;
- if (Val[2] == 0)
- Val[2] = 1;
- };
- CorrectMultiDim(KernelArgs->UserThreadLimit);
- CorrectMultiDim(KernelArgs->UserNumBlocks);
+ correctMultiDim(*KernelArgs);
// Version 3 put the implicit argument at the front with no storage.
if (KernelArgs->Version == OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR) {
@@ -675,6 +689,289 @@ EXTERN void *__tgt_get_mapped_ptr(int64_t DeviceId, const void *HostPtr) {
return TargetPtr;
}
+//===----------------------------------------------------------------------===//
+// Taskgraph transmission (libomp -> libomptarget)
+//
+// The taskgraph is transmitted twice by libomp: the first time to measure the
+// total needed size, the second to do the transfer into a single allocated
+// block of that size.
+//===----------------------------------------------------------------------===//
+
+using llvm::omp::target::TaskGraphBuilder;
+using llvm::omp::target::TaskGraphEdge;
+using llvm::omp::target::TaskGraphElement;
+using llvm::omp::target::TaskGraphExclRegion;
+using llvm::omp::target::TaskGraphIrreducibleRegion;
+using llvm::omp::target::TaskGraphNode;
+using llvm::omp::target::TaskGraphParRegion;
+using llvm::omp::target::TaskGraphRegion;
+using llvm::omp::target::TaskGraphSeqRegion;
+using llvm::omp::target::TaskGraphTargetDataNode;
+using llvm::omp::target::TaskGraphTargetEnterDataNode;
+using llvm::omp::target::TaskGraphTargetExitDataNode;
+using llvm::omp::target::TaskGraphTargetNode;
+using llvm::omp::target::TaskGraphTargetUpdateDataNode;
+using llvm::omp::target::TaskGraphTy;
+using llvm::omp::target::TGKind;
+
+namespace {
+
+/// Resolve the (possibly default) device id used to host a graph.
+static int64_t resolveGraphDevice(int64_t DeviceId) {
+ return DeviceId == OFFLOAD_DEVICE_DEFAULT ? omp_get_default_device()
+ : DeviceId;
+}
+
+/// Append a target enter/exit/update data leaf to the graph.
+template <class NodeT>
+static void emitTaskGraphData(void *Graph, int64_t DeviceId, int32_t ArgNum,
+ void **ArgsBase, void **Args, int64_t *ArgSizes,
+ int64_t *ArgTypes, void **ArgNames,
+ void **ArgMappers) {
+ TaskGraphBuilder B(*static_cast<TaskGraphTy *>(Graph));
+ NodeT *N = B.makeLeaf<NodeT>();
+ if (!N)
+ return;
+ N->DeviceId = DeviceId;
+ N->ArgNum = ArgNum;
+ N->ArgsBase = ArgsBase;
+ N->Args = Args;
+ N->ArgSizes = ArgSizes;
+ N->ArgTypes = ArgTypes;
+ N->ArgNames = ArgNames;
+ N->ArgMappers = ArgMappers;
+ B.linkChild(N);
+}
+
+/// Open a region of \p NodeCount children: the measuring pass sizes it and its
+/// child array, the build pass constructs it and fills it with the children
+/// that follow.
+template <class RegionT>
+static void startRegion(void *Graph, int32_t NodeCount) {
+ TaskGraphBuilder B(*static_cast<TaskGraphTy *>(Graph));
+ if (auto *R = B.makeRegion<RegionT>(NodeCount))
+ B.pushRegion(R);
+}
+
+/// Close the region being filled.
+static void endRegion(void *Graph) {
+ TaskGraphBuilder B(*static_cast<TaskGraphTy *>(Graph));
+ if (B.building())
+ B.popRegion();
+}
+
+} // namespace
+
+EXTERN void *__tgt_taskgraph_start(int64_t DeviceId, uintptr_t GraphId,
+ size_t ByteSize) {
+ assert(PM && "Runtime not initialized");
+ // ByteSize == 0 is the measuring pass (accumulate size only); ByteSize > 0 is
+ // the build pass.
+ return TaskGraphBuilder::start(DeviceId, GraphId, ByteSize);
+}
+
+EXTERN void __tgt_taskgraph_start_parallel(void *Graph, int32_t NodeCount) {
+ startRegion<TaskGraphParRegion>(Graph, NodeCount);
+}
+EXTERN void __tgt_taskgraph_end_parallel(void *Graph) { endRegion(Graph); }
+EXTERN void __tgt_taskgraph_start_sequential(void *Graph, int32_t NodeCount) {
+ startRegion<TaskGraphSeqRegion>(Graph, NodeCount);
+}
+EXTERN void __tgt_taskgraph_end_sequential(void *Graph) { endRegion(Graph); }
+EXTERN void __tgt_taskgraph_start_exclusive(void *Graph, int32_t NodeCount) {
+ startRegion<TaskGraphExclRegion>(Graph, NodeCount);
+}
+EXTERN void __tgt_taskgraph_end_exclusive(void *Graph) { endRegion(Graph); }
+EXTERN void __tgt_taskgraph_start_irreducible(void *Graph, int32_t NodeCount,
+ int32_t EdgeCount) {
+ TaskGraphBuilder B(*static_cast<TaskGraphTy *>(Graph));
+ // The edges, filled by the emit_edge calls that follow the children.
+ if (auto *R = B.makeIrreducibleRegion(NodeCount, EdgeCount))
+ B.pushRegion(R);
+}
+EXTERN void __tgt_taskgraph_end_irreducible(void *Graph) { endRegion(Graph); }
+EXTERN void __tgt_taskgraph_emit_edge(void *Graph, int32_t SrcChild,
+ int32_t DstChild) {
+ // Edges are sized up front with the region's edge array (start_irreducible),
+ // so the measuring pass has nothing to do here. On the build pass they land
+ // in the enclosing irreducible region, which the cursor is currently filling.
+ TaskGraphBuilder B(*static_cast<TaskGraphTy *>(Graph));
+ if (B.building())
+ B.addEdge(SrcChild, DstChild);
+}
+
+EXTERN void __tgt_taskgraph_emit_target(void *Graph, int64_t DeviceId,
+ int32_t NumTeams, int32_t ThreadLimit,
+ void *HostPtr, void *KernelArgs) {
+ TaskGraphBuilder B(*static_cast<TaskGraphTy *>(Graph));
+ auto *N = B.makeLeaf<TaskGraphTargetNode>();
+ if (!N)
+ return;
+ N->DeviceId = DeviceId;
+ N->NumTeams = NumTeams;
+ N->ThreadLimit = ThreadLimit;
+ N->HostPtr = HostPtr;
+ N->KernelArgs = KernelArgs;
+ B.linkChild(N);
+}
+
+EXTERN void __tgt_taskgraph_emit_target_enter_data(
+ void *Graph, int64_t DeviceId, int32_t ArgNum, void **ArgsBase, void **Args,
+ int64_t *ArgSizes, int64_t *ArgTypes, void **ArgNames, void **ArgMappers) {
+ emitTaskGraphData<TaskGraphTargetEnterDataNode>(
+ Graph, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames,
+ ArgMappers);
+}
+
+EXTERN void __tgt_taskgraph_emit_target_exit_data(
+ void *Graph, int64_t DeviceId, int32_t ArgNum, void **ArgsBase, void **Args,
+ int64_t *ArgSizes, int64_t *ArgTypes, void **ArgNames, void **ArgMappers) {
+ emitTaskGraphData<TaskGraphTargetExitDataNode>(
+ Graph, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames,
+ ArgMappers);
+}
+
+EXTERN void __tgt_taskgraph_emit_target_update(
+ void *Graph, int64_t DeviceId, int32_t ArgNum, void **ArgsBase, void **Args,
+ int64_t *ArgSizes, int64_t *ArgTypes, void **ArgNames, void **ArgMappers) {
+ emitTaskGraphData<TaskGraphTargetUpdateDataNode>(
+ Graph, DeviceId, ArgNum, ArgsBase, Args, ArgSizes, ArgTypes, ArgNames,
+ ArgMappers);
+}
+
+EXTERN size_t __tgt_taskgraph_end(void *Graph) {
+ auto *G = static_cast<TaskGraphTy *>(Graph);
+ TaskGraphBuilder B(*G);
+ const bool Measuring = !B.building();
+ size_t Bytes = B.finish();
+ // The measuring pass exists only to compute the block size: libomp feeds it
+ // back to __tgt_taskgraph_start and re-streams the graph on a build pass, so
+ // this throwaway handle is discarded here. The build pass keeps the handle:
+ // its block is fully built, ready to be handed to a plugin.
+ if (Measuring)
+ delete G;
+ return Bytes;
+}
+
+//===----------------------------------------------------------------------===//
+// Services a plugin needs while lowering a graph
+//
+// The libomptarget-only operations a plugin cannot perform for itself, handed
+// over as function pointers in TaskGraphServicesTy because a plugin must not
+// name a libomptarget symbol: the plugin archives are linked into LLVMOffload
+// as well as into libomptarget.so, and LLVMOffload contains none of this.
+//===----------------------------------------------------------------------===//
+
+/// TaskGraphServicesTy::resolveKernel.
+static void *taskGraphResolveKernel(int64_t DeviceId, void *HostPtr) {
+ return getDeviceKernelEntry(static_cast<int32_t>(DeviceId), HostPtr);
+}
+
+/// Why \p G cannot be offloaded to \p DeviceId, or null if it can, settling
+/// what every backend would otherwise check for itself. Also resolves the
+/// default device across the tree, and settles each recorded launch's
+/// configuration as targetKernel does.
+static const char *whyNotOffloadable(TaskGraphTy &G, int64_t DeviceId) {
+ if (const char *Why = TaskGraphBuilder::whyMalformed(G))
+ return Why;
+
+ // Safe to do in place: finalize runs once, on the build-pass handle, before
+ // anything reads the leaves.
+ G.DeviceId = DeviceId;
+ for (TaskGraphNode *N : G.leaves()) {
+ if (N->DeviceId == OFFLOAD_DEVICE_DEFAULT)
+ N->DeviceId = DeviceId;
+ // A backend lowers the graph for the one device it is offered on.
+ if (N->DeviceId != DeviceId)
+ return "its leaves target more than one device";
+
+ auto *K = llvm::dyn_cast<TaskGraphTargetNode>(N);
+ if (!K)
+ continue;
+ // The recording is libomp's own copy, and host replay goes through
+ // targetKernel, which makes the same adjustments again.
+ auto *KA = static_cast<KernelArgsTy *>(K->KernelArgs);
+ if (!KA)
+ return "a target leaf with no recorded kernel arguments";
+ // Version 3 put the launch environment first, with no slot recorded for
+ // it, which only upgradeKernelArgs's per-launch copy straightens out.
+ if (KA->Version <= OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR)
+ return "a target leaf built for an older kernel-argument ABI";
+ if (K->NumTeams == -1)
+ KA->UserNumBlocks[0] = 1;
+ correctMultiDim(*KA);
+ }
+ return nullptr;
+}
+
+EXTERN int __tgt_taskgraph_finalize(void *Graph) {
+ auto *G = static_cast<TaskGraphTy *>(Graph);
+ int64_t ResolvedDevice = resolveGraphDevice(G->DeviceId);
+ auto DeviceOrErr = PM->getDevice(ResolvedDevice);
+ if (!DeviceOrErr) {
+ consumeError(DeviceOrErr.takeError());
+ return OFFLOAD_FAIL;
+ }
+ DeviceTy &Device = *DeviceOrErr;
+
+ if (const char *Why = whyNotOffloadable(*G, ResolvedDevice)) {
+ INFO(OMP_INFOTYPE_ALL, Device.DeviceID,
+ "taskgraph %" PRIu64 " replays on the host: %s\n",
+ static_cast<uint64_t>(G->GraphId), Why);
+ return OFFLOAD_FAIL;
+ }
+
+ G->Services.resolveKernel = taskGraphResolveKernel;
+
+ // Offer the tree to the plugin, which lowers it however suits its backend.
+ // The base plugin implementation declines, as does a backend that meets
+ // something in the graph it cannot express; either way libomp keeps the graph
+ // and replays it on the host.
+ void *PluginGraph = nullptr;
+ if (Device.RTL->finalize_taskgraph(Device.RTLDeviceID, G, &PluginGraph) !=
+ OFFLOAD_SUCCESS)
+ return OFFLOAD_FAIL;
+ assert(PluginGraph && "plugin claimed a taskgraph without a handle");
+ G->PluginGraph = PluginGraph;
+ return OFFLOAD_SUCCESS;
+}
+
+EXTERN void __tgt_taskgraph_begin_replay(void *Graph) {
+ // Nothing a graph holds yet depends on when the replay started.
+ assert(static_cast<TaskGraphTy *>(Graph)->PluginGraph &&
+ "replaying a graph no plugin claimed");
+}
+
+EXTERN int __tgt_taskgraph_replay(void *Graph) {
+ auto *G = static_cast<TaskGraphTy *>(Graph);
+ assert(G->PluginGraph && "replaying a graph no plugin claimed");
+
+ auto DeviceOrErr = PM->getDevice(G->DeviceId);
+ if (!DeviceOrErr)
+ FATAL_MESSAGE(G->DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());
+ DeviceTy &Device = *DeviceOrErr;
+
+ // The backend issues the graph and waits for it. Part of the graph may have
+ // run by the time it fails, so a failure cannot fall back to the host.
+ if (Device.RTL->replay_taskgraph(Device.RTLDeviceID, G) != OFFLOAD_SUCCESS)
+ FATAL_MESSAGE(G->DeviceId, "%s", "taskgraph replay failed");
+ return OFFLOAD_SUCCESS;
+}
+
+EXTERN void __tgt_taskgraph_destroy(void *Graph) {
+ auto *G = static_cast<TaskGraphTy *>(Graph);
+ if (G->PluginGraph) {
+ // DeviceId was resolved to a concrete device by finalize, which is also the
+ // only thing that can set PluginGraph.
+ auto DeviceOrErr = PM->getDevice(G->DeviceId);
+ if (DeviceOrErr)
+ DeviceOrErr->RTL->destroy_taskgraph(DeviceOrErr->RTLDeviceID, G);
+ else
+ consumeError(DeviceOrErr.takeError());
+ }
+ delete G;
+}
+
EXTERN void __tgt_taskgraph_dup_kernel_args(void *Dst, void *Src,
size_t *AllocSize) {
// Deep-copy a kernel-arguments blob into a single caller-provided block, so
@@ -786,6 +1083,7 @@ EXTERN void __tgt_taskgraph_dup_data_args(void *Dst, int32_t ArgNum,
// that size, which is when the in-out array parameters are repointed into it.
bool SizeOnly = (Dst == nullptr);
size_t N = ArgNum;
+ namespace mt = omptarget::maptype;
// Copying the arrays is not enough on its own for a non-contiguous update.
// Its args[i] is a pointer to a stride descriptor rather than to mapped
// data, and clang emits that descriptor into the encountering function's
@@ -797,9 +1095,7 @@ EXTERN void __tgt_taskgraph_dup_data_args(void *Dst, int32_t ArgNum,
// rewrite. arg_sizes[i] is the dimension count for such an entry, not a
// byte size.
auto NonContigDims = [&](size_t Idx) -> size_t {
- return ((*ArgTypes)[Idx] & OMP_TGT_MAPTYPE_NON_CONTIG)
- ? (size_t)(*ArgSizes)[Idx]
- : 0;
+ return mt::isNonContig((*ArgTypes)[Idx]) ? (size_t)(*ArgSizes)[Idx] : 0;
};
// ArgNames and ArgMappers are optional; a construct with neither debug names
// nor user-defined mappers leaves them null, and the copy has to preserve
diff --git a/offload/libomptarget/omptarget.cpp b/offload/libomptarget/omptarget.cpp
index 0038796447f84b..a1555f1111b4b9 100644
--- a/offload/libomptarget/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -2354,6 +2354,24 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
return OFFLOAD_SUCCESS;
}
+/// Resolve the device-side kernel entry for a host function pointer (see
+/// private.h). Mirrors the table lookup target() performs before launch, but
+/// without touching any data mappings -- the taskgraph backend only needs the
+/// device entry handle at finalize. getTableMap has internal linkage but is in
+/// scope here (same translation unit).
+void *getDeviceKernelEntry(int32_t DeviceId, void *HostPtr) {
+ TableMap *TM = getTableMap(HostPtr);
+ if (!TM)
+ return nullptr;
+ std::lock_guard<std::mutex> TrlTblLock(PM->TrlTblMtx);
+ if (TM->Table->TargetsTable.size() <= static_cast<size_t>(DeviceId))
+ return nullptr;
+ __tgt_target_table *TargetTable = TM->Table->TargetsTable[DeviceId];
+ if (!TargetTable)
+ return nullptr;
+ return TargetTable->EntriesBegin[TM->Index].Address;
+}
+
/// Enables the record replay mechanism by pre-allocating MemorySize
/// and informing the record-replayer of whether to store the output
/// in some file.
diff --git a/offload/libomptarget/private.h b/offload/libomptarget/private.h
index 641701627cfcd4..fa88c40d1cf5ab 100644
--- a/offload/libomptarget/private.h
+++ b/offload/libomptarget/private.h
@@ -42,6 +42,12 @@ target_replay(ident_t *Loc, DeviceTy &Device, void *HostPtr, void *DeviceMemory,
extern void handleTargetOutcome(bool Success, ident_t *Loc);
+/// Resolve the device-side kernel entry (a plugin \c GenericKernelTy*) for the
+/// host function pointer \p HostPtr on libomptarget device \p DeviceId, or
+/// return nullptr if no matching entry / target table exists. Used by the
+/// taskgraph backend to fill a kernel node's device handle at finalize.
+extern void *getDeviceKernelEntry(int32_t DeviceId, void *HostPtr);
+
////////////////////////////////////////////////////////////////////////////////
/// Print out the names and properties of the arguments to each kernel
static inline void
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 5e2a671a0baa2a..590838efef374d 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -28,6 +28,7 @@
#include "Shared/Environment.h"
#include "Shared/EnvironmentVar.h"
#include "Shared/Requirements.h"
+#include "Shared/TaskGraph.h"
#include "Shared/Utils.h"
#include "GenericProfiler.h"
@@ -1266,6 +1267,33 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
RecordReplayTy *getRecordReplay() { return RecordReplay; }
+ /// Taskgraph handoff hooks. \p Graph is a shared taskgraph tree
+ /// (Shared/TaskGraph.h) of target constructs only, which the plugin only
+ /// reads. libomptarget only offers a well-formed graph: every region has the
+ /// children and edges it announced, irreducible regions are acyclic, every
+ /// leaf targets the graph's device, and every target leaf has a recorded
+ /// launch whose configuration is settled as for a launch outside a graph.
+ ///
+ /// finalizeTaskGraph interprets the tree into whatever executable form suits
+ /// the backend and claims ownership of the graph by returning that form,
+ /// which must not be null; libomptarget keeps it in the graph's PluginGraph,
+ /// where the other hooks find it. replay dispatches that form and waits for
+ /// it to complete, and destroy frees it. The base implementations decline,
+ /// as should any backend that meets something in the tree it cannot express.
+ /// libomp runs the target constructs of a declined graph itself.
+ virtual Expected<void *>
+ finalizeTaskGraph(const llvm::omp::target::TaskGraphTy *Graph) {
+ return Plugin::error(error::ErrorCode::UNSUPPORTED,
+ "taskgraph finalize unsupported by this plugin");
+ }
+ virtual Error replayTaskGraph(const llvm::omp::target::TaskGraphTy *Graph) {
+ return Plugin::error(error::ErrorCode::UNSUPPORTED,
+ "taskgraph replay unsupported by this plugin");
+ }
+ virtual Error destroyTaskGraph(const llvm::omp::target::TaskGraphTy *Graph) {
+ return Plugin::success();
+ }
+
/// Map to record kernel have been launchedl, for error reporting purposes.
ProtectedObj<KernelTraceInfoRecordTy> KernelLaunchTraces;
@@ -1567,6 +1595,19 @@ struct GenericPluginTy {
__tgt_async_info *AsyncInfoPtr,
GenericProfilerTy *ProfilerPtr = nullptr);
+ /// Taskgraph support: build an executable form of \p Graph, replay it, or
+ /// destroy it on the given device. Return OFFLOAD_SUCCESS only if the plugin
+ /// handled the request; libomp replays the graph on the host otherwise. A
+ /// successful finalize returns the plugin's form of the graph in
+ /// \p PluginGraph.
+ int32_t finalize_taskgraph(int32_t DeviceId,
+ const llvm::omp::target::TaskGraphTy *Graph,
+ void **PluginGraph);
+ int32_t replay_taskgraph(int32_t DeviceId,
+ const llvm::omp::target::TaskGraphTy *Graph);
+ int32_t destroy_taskgraph(int32_t DeviceId,
+ const llvm::omp::target::TaskGraphTy *Graph);
+
/// Returns if the plugin can support automatic copy.
int32_t use_auto_zero_copy(int32_t DeviceId);
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 2e07d343574f80..73dee99ce39e87 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -1231,6 +1231,49 @@ int32_t GenericPluginTy::launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
return OFFLOAD_SUCCESS;
}
+int32_t
+GenericPluginTy::finalize_taskgraph(int32_t DeviceId,
+ const llvm::omp::target::TaskGraphTy *Graph,
+ void **PluginGraph) {
+ auto PluginGraphOrErr = getDevice(DeviceId).finalizeTaskGraph(Graph);
+ if (Error Err = PluginGraphOrErr.takeError()) {
+ // Declining is the normal case for plugins without a graph backend, and a
+ // legitimate outcome for one that meets a construct it cannot express;
+ // libomp then replays the graph on the host. So this is not reported as a
+ // failure -- but the reason is the first thing anyone asks when a graph
+ // unexpectedly runs on the host, so it goes to the debug stream.
+ std::string Reason = toString(std::move(Err));
+ ODBG(OLDT_Module) << "device " << DeviceId << " declined taskgraph "
+ << Graph << ": " << Reason;
+ return OFFLOAD_FAIL;
+ }
+ *PluginGraph = *PluginGraphOrErr;
+ return OFFLOAD_SUCCESS;
+}
+
+int32_t
+GenericPluginTy::replay_taskgraph(int32_t DeviceId,
+ const llvm::omp::target::TaskGraphTy *Graph) {
+ auto Err = getDevice(DeviceId).replayTaskGraph(Graph);
+ if (Err) {
+ REPORT() << "Failure to replay taskgraph on device " << DeviceId << ": "
+ << toString(std::move(Err));
+ return OFFLOAD_FAIL;
+ }
+ return OFFLOAD_SUCCESS;
+}
+
+int32_t GenericPluginTy::destroy_taskgraph(
+ int32_t DeviceId, const llvm::omp::target::TaskGraphTy *Graph) {
+ auto Err = getDevice(DeviceId).destroyTaskGraph(Graph);
+ if (Err) {
+ REPORT() << "Failure to destroy taskgraph on device " << DeviceId << ": "
+ << toString(std::move(Err));
+ return OFFLOAD_FAIL;
+ }
+ return OFFLOAD_SUCCESS;
+}
+
int32_t GenericPluginTy::use_auto_zero_copy(int32_t DeviceId) {
return getDevice(DeviceId).useAutoZeroCopy();
}
>From 5598901a3458534310115f031dfc7850da7dad14 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 19 Aug 2026 10:42:43 -0500
Subject: [PATCH 19/29] [OpenMP] Transmission of taskgraphs,
libomp->libomptarget: libomp side
This is the libomp part of the transmission protocol for graphs from
libomp to libomptarget. The libomptarget part is in the previous patch,
"[Offload] Transmission of taskgraphs, libomp->libomptarget: omptarget
side".
The graph is constructed on the libomptarget side using a set of functions
describing the structure of the graph, which libomp looks up at run time
with KMP_DLSYM rather than linking against libomptarget.
The actual transmission is done by a walk over the region tree in
__kmp_taskgraph_transmit_to_target, which offers every maximal subtree
holding only target constructs to libomptarget as a subgraph of its own,
and keeps the handle of each one a plugin claims on the subtree's root
region. A target construct whose 'mutexinoutset' survives region
building is never put in a subgraph, nor is a parallel region with task
reductions: libomp implements both, so it keeps running those itself.
On replay, a region holding a subgraph is a single step of the flattened
exec descriptor DAG, as a target construct is: the exec descriptors are
built and released exactly as before, and when a subgraph's step is
reached, libomp has libomptarget run it and waits for it to finish before
releasing its successors. If libomptarget declines to run it on some
replay -- for a mapping state its plan cannot express, say -- nothing has
been issued, and libomp re-issues the subgraph's target constructs one at
a time in an order its tree allows. libomp relocates every recorded node
when the replay starts, host tasks and target constructs alike, then has
libomptarget begin a replay of every subgraph.
Tests have been added to cover new functionality, though some won't pass
until the later patch in the series that adds target graph replay to an
offload plugin, so are XFAILed here.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/225718
---
.../taskgraph_host_task_subgraph_order.c | 199 ++++++++
.../taskgraph_target_mutex_exclusive.c | 97 ++++
.../taskgraph_target_mutex_irreducible.c | 123 +++++
.../taskgraph_target_replay_mixed.c | 5 +-
openmp/runtime/src/kmp.h | 16 +
openmp/runtime/src/kmp_taskdeps.cpp | 6 +
openmp/runtime/src/kmp_tasking.cpp | 438 +++++++++++++++++-
7 files changed, 866 insertions(+), 18 deletions(-)
create mode 100644 offload/test/offloading/taskgraph_host_task_subgraph_order.c
create mode 100644 offload/test/offloading/taskgraph_target_mutex_exclusive.c
create mode 100644 offload/test/offloading/taskgraph_target_mutex_irreducible.c
diff --git a/offload/test/offloading/taskgraph_host_task_subgraph_order.c b/offload/test/offloading/taskgraph_host_task_subgraph_order.c
new file mode 100644
index 00000000000000..2d6c0e910849b2
--- /dev/null
+++ b/offload/test/offloading/taskgraph_host_task_subgraph_order.c
@@ -0,0 +1,199 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env KMP_TASKGRAPH_TRACE=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefixes=CHECK,TRACE \
+// RUN: --implicit-check-not='replays on the host'
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// Host tasks that call into the offload runtime themselves, ordered against
+// target constructs in the same graph. Host tasks never enter libomptarget:
+// libomp runs them itself, and hands each maximal target-only part of the
+// graph to libomptarget as a subgraph of its own, which runs as a single step
+// of the replay and has finished when that step returns. The ordering between
+// a host task and a subgraph is therefore libomp's to keep, and that is what
+// this tests: every shape checks its data, and a lost edge is a wrong number.
+//
+// It is a liveness test as well. Because a subgraph is issued only once the
+// host tasks before it have finished, and a host task starts only once the
+// subgraphs before it have run, target work is never queued on the device
+// behind a host task. The first two shapes are the ones that hang if that
+// ever stops being true: a copy ordered after a host task would stall the
+// device's one in-order blit queue, which a copy the task itself issues then
+// queues behind; and a host task run as a device-stream callback would run on
+// the HSA runtime's async handler thread, which an unpinned device-to-host copy
+// from the task then needs.
+//
+// * Shape 1: a `target update from`, a subgraph of its own, is ordered after a
+// task that writes the device buffer.
+// * Shape 2: nothing follows the task, but it runs after a subgraph -- a
+// `target update to` and a kernel -- and copies device-to-host.
+// * Shape 3, the join, `seq(par(H1, H2), more)`: two independent tasks, then
+// a subgraph whose kernel names both of them, so it may only run once both
+// have. The result is only right if the kernel ran after both tasks had
+// finished, not merely started.
+//
+// The first run turns the AMDGPU graph backend off, so that the plugin claims
+// no subgraph and libomp issues each target construct itself; the others let
+// the plugin replay the subgraphs, and the traced run checks that it does.
+//
+// All three are then run again from inside a `parallel` region, which is not
+// optional coverage: at top level the team is serialized, so every host task
+// is run by the encountering thread. A task run by another thread, and
+// completing while the encountering thread is waiting, is only reached with a
+// real team.
+//
+// The device buffers are allocated and associated with their host arrays
+// before any graph runs, so each graph finds them already mapped. A host task
+// therefore gets its device address by capture, and the copy-outs are
+// `target update`s.
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 256
+#define HALF (N / 2)
+#define ITERS 4
+
+static int Dev, Host;
+
+// Run every shape ITERS times over its own buffers, accumulating mismatch
+// counts. Called once at top level and once from a parallel region, so it
+// keeps no state of its own beyond the buffers the caller owns. DevA, DevB
+// and DevC are the device buffers associated with A, B and C.
+static void runShapes(int *Src, int *A, int *B, int *C, int *Out, int *DevA,
+ int *DevB, int *DevC, int *BadCopyOut, int *BadReadBack,
+ int *BadJoin) {
+ int Gate = 0, Gate1 = 0, Gate2 = 0, GateB = 0, GateC = 0;
+
+ for (int It = 0; It < ITERS; ++It) {
+ for (int I = 0; I < N; ++I) {
+ Src[I] = 100 * (It + 1) + I;
+ A[I] = -1;
+ B[I] = 1000 * (It + 1) + I;
+ C[I] = -1;
+ Out[I] = -1;
+ }
+
+ // Shape 1: a copy-out is ordered after a host task that writes the device
+ // buffer.
+#pragma omp taskgraph
+ {
+#pragma omp task firstprivate(DevA) shared(Src) depend(out : Gate)
+ omp_target_memcpy(DevA, Src, N * sizeof(int), 0, 0, Dev, Host);
+#pragma omp target update from(A[0 : N]) depend(in : Gate)
+ }
+
+ // Shape 2: nothing follows the host task, but it runs after a kernel on
+ // the same graph and reads the result back device-to-host.
+#pragma omp taskgraph
+ {
+#pragma omp target update to(B[0 : N]) depend(out : GateB)
+#pragma omp target is_device_ptr(DevB) depend(inout : GateB)
+ for (int I = 0; I < N; ++I)
+ DevB[I] += 1;
+#pragma omp task firstprivate(DevB) shared(Out) depend(in : GateB)
+ omp_target_memcpy(Out, DevB, N * sizeof(int), 0, 0, Host, Dev);
+ }
+
+ // Shape 3: the join. Two independent host tasks fill one half of the
+ // device buffer each, and a kernel that names both adds to the whole of
+ // it, so the sum below is only right if the kernel ran after both.
+#pragma omp taskgraph
+ {
+#pragma omp task firstprivate(DevC) shared(Src) depend(out : Gate1)
+ omp_target_memcpy(DevC, Src, HALF * sizeof(int), 0, 0, Dev, Host);
+#pragma omp task firstprivate(DevC) shared(Src) depend(out : Gate2)
+ omp_target_memcpy(DevC, Src, HALF * sizeof(int), HALF * sizeof(int),
+ HALF * sizeof(int), Dev, Host);
+#pragma omp target is_device_ptr(DevC) depend(in : Gate1, Gate2) \
+ depend(out : GateC)
+ for (int I = 0; I < N; ++I)
+ DevC[I] += 7;
+#pragma omp target update from(C[0 : N]) depend(in : GateC)
+ }
+
+ for (int I = 0; I < N; ++I) {
+ if (A[I] != 100 * (It + 1) + I)
+ ++*BadCopyOut;
+ if (Out[I] != 1000 * (It + 1) + I + 1)
+ ++*BadReadBack;
+ if (C[I] != 100 * (It + 1) + I + 7)
+ ++*BadJoin;
+ }
+ }
+}
+
+int main() {
+ Dev = omp_get_default_device();
+ Host = omp_get_initial_device();
+ int *Src = (int *)malloc(N * sizeof(int));
+ int *A = (int *)malloc(N * sizeof(int));
+ int *B = (int *)malloc(N * sizeof(int));
+ int *C = (int *)malloc(N * sizeof(int));
+ int *Out = (int *)malloc(N * sizeof(int));
+ int *DevA = (int *)omp_target_alloc(N * sizeof(int), Dev);
+ int *DevB = (int *)omp_target_alloc(N * sizeof(int), Dev);
+ int *DevC = (int *)omp_target_alloc(N * sizeof(int), Dev);
+ omp_target_associate_ptr(A, DevA, N * sizeof(int), 0, Dev);
+ omp_target_associate_ptr(B, DevB, N * sizeof(int), 0, Dev);
+ omp_target_associate_ptr(C, DevC, N * sizeof(int), 0, Dev);
+
+ int BadCopyOut = 0, BadReadBack = 0, BadJoin = 0;
+ runShapes(Src, A, B, C, Out, DevA, DevB, DevC, &BadCopyOut, &BadReadBack,
+ &BadJoin);
+
+ // The target constructs go to the device.
+ // TRACE: Transmitted taskgraph {{.*}} subgraph
+ // CHECK: copy-out did not see the task's writes: 0
+ printf("copy-out did not see the task's writes: %d\n", BadCopyOut);
+ // CHECK: read-back did not see the kernel's writes: 0
+ printf("read-back did not see the kernel's writes: %d\n", BadReadBack);
+ // CHECK: joining kernel did not see both tasks' writes: 0
+ printf("joining kernel did not see both tasks' writes: %d\n", BadJoin);
+
+ // Again with a team, so the host tasks may run on other threads. `single`
+ // because the graph is per-encounter; the other threads go to its implicit
+ // barrier and are available to steal the tasks from there.
+ int ParBadCopyOut = 0, ParBadReadBack = 0, ParBadJoin = 0;
+#pragma omp parallel num_threads(4)
+ {
+#pragma omp single
+ runShapes(Src, A, B, C, Out, DevA, DevB, DevC, &ParBadCopyOut,
+ &ParBadReadBack, &ParBadJoin);
+ }
+
+ // CHECK: parallel copy-out did not see the task's writes: 0
+ printf("parallel copy-out did not see the task's writes: %d\n",
+ ParBadCopyOut);
+ // CHECK: parallel read-back did not see the kernel's writes: 0
+ printf("parallel read-back did not see the kernel's writes: %d\n",
+ ParBadReadBack);
+ // CHECK: parallel joining kernel did not see both tasks' writes: 0
+ printf("parallel joining kernel did not see both tasks' writes: %d\n",
+ ParBadJoin);
+
+ omp_target_disassociate_ptr(C, Dev);
+ omp_target_disassociate_ptr(B, Dev);
+ omp_target_disassociate_ptr(A, Dev);
+ omp_target_free(DevC, Dev);
+ omp_target_free(DevB, Dev);
+ omp_target_free(DevA, Dev);
+ free(Out);
+ free(C);
+ free(B);
+ free(A);
+ free(Src);
+ return (BadCopyOut + BadReadBack + BadJoin + ParBadCopyOut + ParBadReadBack +
+ ParBadJoin) != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_mutex_exclusive.c b/offload/test/offloading/taskgraph_target_mutex_exclusive.c
new file mode 100644
index 00000000000000..b3f9950030c3d7
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_mutex_exclusive.c
@@ -0,0 +1,97 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env KMP_TASKGRAPH_TRACE=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic \
+// RUN: --implicit-check-not='sets:'
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: env KMP_TASKGRAPH_TRACE=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic \
+// RUN: --implicit-check-not='sets:'
+
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// `mutexinoutset` on a target construct, in the shape where the region builder
+// can express the exclusion structurally. The three targets are twins -- same
+// (empty) predecessor and successor sets -- so the merge pass folds them into a
+// single EXCLUSIVE region, and __kmp_taskgraph_strip_mutex_sets then drops their
+// mutex sets, the region itself being what carries the exclusion from there on.
+//
+// That strip is what this test is really about, because it is what leaves the
+// targets offloadable: libomp keeps a target that still carries a set out of
+// every subgraph and issues it itself under the recorded locks, since a device
+// schedule has no way to express "unordered but not simultaneous". With the
+// sets gone the leaves are ordinary, the exclusive region can be offered to
+// libomptarget as a subgraph, and the backend serializes it exactly as it does
+// a sequential one -- a total order being a trivially correct way to satisfy
+// mutual exclusion. Hence the implicit check-not: no set may survive into the
+// trace.
+//
+// Whether the plugin goes on to claim the subgraph is a property of the build
+// rather than of the strip, so it is not asserted: a plugin with no taskgraph
+// lowering declines every graph (the base finalizeTaskGraph returns
+// UNSUPPORTED) and libomp replays the targets itself, which is a correct
+// outcome that still has to produce the answer below. The trace is only
+// required to show the subgraph reaching the handoff at all, so that a build
+// which never offers it -- the "handoff ABI is unavailable" path -- is still
+// caught.
+//
+// A target node has no task, so this also checks that recording keeps a
+// target's `mutexinoutset` rather than treating it as `inout`, which would
+// serialize the three in encounter order and produce `sequential` here.
+//
+// The buffer is shared and updated non-atomically by all three, so the expected
+// value depends on the exclusion holding; `nowait` is what leaves them
+// unordered, without which encounter order would order them anyway and the
+// exclusive region would never be built.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 256;
+ const int Iters = 3;
+ int Mtx;
+ int *s = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ s[i] = i;
+
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target nowait map(tofrom : s[0 : N]) depend(mutexinoutset : Mtx)
+ {
+ for (int i = 0; i < N; ++i)
+ s[i] += 1;
+ }
+#pragma omp target nowait map(tofrom : s[0 : N]) depend(mutexinoutset : Mtx)
+ {
+ for (int i = 0; i < N; ++i)
+ s[i] += 2;
+ }
+#pragma omp target nowait map(tofrom : s[0 : N]) depend(mutexinoutset : Mtx)
+ {
+ for (int i = 0; i < N; ++i)
+ s[i] += 4;
+ }
+ }
+ }
+
+ // Each encounter adds 1 + 2 + 4 in some order, so exactly 7.
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (s[i] != i + Iters * 7)
+ ++errors;
+
+ // CHECK: Processed taskgraph
+ // CHECK-NEXT: exclusive {
+ // CHECK-NEXT: target: 0x{{[[:xdigit:]]+}}
+ // CHECK-NEXT: target: 0x{{[[:xdigit:]]+}}
+ // CHECK-NEXT: target: 0x{{[[:xdigit:]]+}}
+ // CHECK-NEXT: }
+ // CHECK: {{Transmitted taskgraph|was declined by the device plugin}}
+
+ // CHECK: taskgraph target mutex exclusive errors: 0
+ printf("taskgraph target mutex exclusive errors: %d\n", errors);
+ free(s);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_mutex_irreducible.c b/offload/test/offloading/taskgraph_target_mutex_irreducible.c
new file mode 100644
index 00000000000000..61bb6ff8896e1a
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_mutex_irreducible.c
@@ -0,0 +1,123 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env KMP_TASKGRAPH_TRACE=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic \
+// RUN: --implicit-check-not='exclusive' --implicit-check-not=declined
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: env KMP_TASKGRAPH_TRACE=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic \
+// RUN: --implicit-check-not='exclusive' --implicit-check-not=declined
+
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// The other half of taskgraph_target_mutex_exclusive.c: a `mutexinoutset` on a
+// target that the region builder cannot express structurally, so the mutex sets
+// survive recording and those targets stay on the host.
+//
+// T1 (out: a) T2 (out: b)
+// | \ / |
+// | \ / |
+// TA{m0} TAB{m0,m1} TB{m1}
+// | | |
+// '------ T6 (in: x, y, z) --'
+//
+// The three mutex targets have different predecessor sets (a, b, and both), so
+// the merge pass cannot fold them into one EXCLUSIVE region the way it does for
+// the twins in the exclusive test. Those same differing predecessors make the
+// shape non-series-parallel, so the members land in an IRREDUCIBLE region --
+// where __kmp_taskgraph_strip_mutex_sets recurses without setting in_exclusive
+// and the sets are kept, since nothing else is left to carry the exclusion.
+//
+// A target with a surviving set is never put in a subgraph, a device schedule
+// having no way to express "unordered but not simultaneous", so libomp issues
+// each of the three itself, taking the recorded locks around it. The other
+// targets are offered to libomptarget as subgraphs of their own, and claimed.
+//
+// So the run pins all three: the irreducible shape with no exclusive formed, the
+// sets surviving into replay, and the rest of the graph still going to the
+// device -- plus the answer, which the ordering the mutex targets do have
+// (after T1/T2, before T6) still has to make come out right, so the exclusion
+// cannot be "fixed" by serializing the graph and losing its dependence
+// structure.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 256;
+ const int Iters = 3;
+ int Da, Db, Dm0, Dm1, Dx, Dy, Dz;
+ int *a = (int *)malloc(N * sizeof(int));
+ int *b = (int *)malloc(N * sizeof(int));
+ int *x = (int *)malloc(N * sizeof(int));
+ int *y = (int *)malloc(N * sizeof(int));
+ int *z = (int *)malloc(N * sizeof(int));
+ int *r = (int *)malloc(N * sizeof(int));
+
+ int errors = 0;
+ for (int it = 0; it < Iters; ++it) {
+ for (int i = 0; i < N; ++i)
+ a[i] = b[i] = x[i] = y[i] = z[i] = r[i] = -1;
+
+#pragma omp taskgraph
+ {
+#pragma omp target nowait map(from : a[0 : N]) depend(out : Da)
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = i + 1;
+ }
+#pragma omp target nowait map(from : b[0 : N]) depend(out : Db)
+ {
+ for (int i = 0; i < N; ++i)
+ b[i] = 2 * (i + 1);
+ }
+#pragma omp target nowait map(to : a[0 : N]) map(from : x[0 : N]) \
+ depend(in : Da) depend(mutexinoutset : Dm0) depend(out : Dx)
+ {
+ for (int i = 0; i < N; ++i)
+ x[i] = a[i];
+ }
+#pragma omp target nowait map(to : a[0 : N], b[0 : N]) map(from : z[0 : N]) \
+ depend(in : Da, Db) depend(mutexinoutset : Dm0, Dm1) depend(out : Dz)
+ {
+ for (int i = 0; i < N; ++i)
+ z[i] = a[i] + b[i];
+ }
+#pragma omp target nowait map(to : b[0 : N]) map(from : y[0 : N]) \
+ depend(in : Db) depend(mutexinoutset : Dm1) depend(out : Dy)
+ {
+ for (int i = 0; i < N; ++i)
+ y[i] = b[i];
+ }
+#pragma omp target nowait map(to : x[0 : N], y[0 : N], z[0 : N]) \
+ map(from : r[0 : N]) depend(in : Dx, Dy, Dz)
+ {
+ for (int i = 0; i < N; ++i)
+ r[i] = x[i] + y[i] + z[i];
+ }
+ }
+
+ // x = i+1, y = 2(i+1), z = 3(i+1), so r = 6(i+1).
+ for (int i = 0; i < N; ++i)
+ if (x[i] != i + 1 || y[i] != 2 * (i + 1) || z[i] != 3 * (i + 1) ||
+ r[i] != 6 * (i + 1))
+ ++errors;
+ }
+
+ // CHECK: irreducible {
+ // CHECK: sets:
+ // CHECK: Transmitted taskgraph {{.*}} subgraph
+
+ // CHECK: taskgraph target mutex irreducible errors: 0
+ printf("taskgraph target mutex irreducible errors: %d\n", errors);
+ free(r);
+ free(z);
+ free(y);
+ free(x);
+ free(b);
+ free(a);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_replay_mixed.c b/offload/test/offloading/taskgraph_target_replay_mixed.c
index 668dc2daa3b7c0..ff02124d1c7db9 100644
--- a/offload/test/offloading/taskgraph_target_replay_mixed.c
+++ b/offload/test/offloading/taskgraph_target_replay_mixed.c
@@ -8,8 +8,9 @@
// A mixed host + target 'omp taskgraph', re-encountered several times. A host
// 'task' (depend out) runs before a 'target' kernel (depend in) each encounter.
-// On replay libomp runs the recorded host task and re-issues the target kernel
-// through the standard libomptarget launch path.
+// On replay libomp runs the host task itself and, once it has finished, has
+// libomptarget run the target kernel, a subgraph of its own -- or issues the
+// kernel itself if no plugin claimed the subgraph.
// The dependency ordering (host then device) must be preserved every time.
#include <stdio.h>
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index c6cc4c999fcc72..b2be1c0033678f 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -1171,6 +1171,16 @@ __kmp_get_submemspace(omp_memspace_handle_t memspace, int num_resources,
#if ENABLE_LIBOMPTARGET
extern void __kmp_init_target_task();
+#if OMP_TASKGRAPH_EXPERIMENTAL
+// Offer each maximal subtree of the just-built taskgraph record that holds only
+// target constructs to libomptarget, as a subgraph of its own, and keep the
+// handle of each one a plugin claims on the subtree's root region. No-op if
+// the libomptarget handoff ABI is unavailable. Defined in kmp_tasking.cpp.
+struct kmp_taskgraph_record;
+extern void
+__kmp_taskgraph_transmit_to_target(kmp_int32 gtid,
+ struct kmp_taskgraph_record *rec);
+#endif
#endif
/* ------------------------------------------------------------------------ */
@@ -2812,6 +2822,10 @@ typedef struct kmp_taskgraph_region {
// The exec descr the region is replayed through, once the exec descr array
// has been built.
struct kmp_taskgraph_exec_descr *exec_descr = nullptr;
+ // The libomptarget subgraph this region is the root of, if a plugin claimed
+ // the region (see __kmp_taskgraph_transmit_to_target). A replay then runs
+ // the whole region as one step, on the device.
+ void *tgt_graph = nullptr;
// The exec descrs this region resolves to when a dependency names it, i.e.
// the descrs that must all complete before the region has. Only set while
// the enclosing IRREDUCIBLE region is being built, and only on its direct
@@ -2878,6 +2892,8 @@ typedef struct kmp_taskgraph_record {
struct kmp_taskgraph_exec_descr *exec_descrs = nullptr;
kmp_size_t num_exec_descrs = 0;
void *taskgraph_args = nullptr;
+ // How many of the record's regions hold a libomptarget subgraph.
+ kmp_int32 num_tgt_subgraphs = 0;
// We need a taskgroup structure to keep track of recorded tasks. This is
// set to TRUE if the user requested "nogroup" on the taskgraph directive
// (then we can avoid blocking at the end of the taskgraph region on replay,
diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index f6b7e49a1671a2..d1cdff3274fc9c 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -3343,6 +3343,12 @@ kmp_int32 __kmp_build_taskgraph(kmp_int32 gtid,
// &initial_regions[0], numregions);
#endif
+#if ENABLE_LIBOMPTARGET
+ // Offer the graph's target-only subtrees to libomptarget before publishing
+ // it READY. A replay then has libomptarget run each one a plugin claimed.
+ __kmp_taskgraph_transmit_to_target(gtid, taskgraph);
+#endif
+
KMP_ATOMIC_ST_REL(&taskgraph->status, KMP_TDG_READY);
return 0;
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index b9726894789acd..aa28427abf50e5 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -61,6 +61,35 @@ static void (*tgt_target_data_update_nowait_mapper)(ident_t *, int64_t, int32_t,
int32_t, void *, int32_t,
void *);
+// Taskgraph handoff ABI (see offload/include/omptarget.h).
+static void *(*tgt_taskgraph_start)(int64_t, uintptr_t, size_t);
+static void (*tgt_taskgraph_start_parallel)(void *, int32_t);
+static void (*tgt_taskgraph_end_parallel)(void *);
+static void (*tgt_taskgraph_start_sequential)(void *, int32_t);
+static void (*tgt_taskgraph_end_sequential)(void *);
+static void (*tgt_taskgraph_start_exclusive)(void *, int32_t);
+static void (*tgt_taskgraph_end_exclusive)(void *);
+static void (*tgt_taskgraph_start_irreducible)(void *, int32_t, int32_t);
+static void (*tgt_taskgraph_end_irreducible)(void *);
+static void (*tgt_taskgraph_emit_edge)(void *, int32_t, int32_t);
+static void (*tgt_taskgraph_emit_target)(void *, int64_t, int32_t, int32_t,
+ void *, void *);
+static void (*tgt_taskgraph_emit_target_enter_data)(void *, int64_t, int32_t,
+ void **, void **, int64_t *,
+ int64_t *, void **,
+ void **);
+static void (*tgt_taskgraph_emit_target_exit_data)(void *, int64_t, int32_t,
+ void **, void **, int64_t *,
+ int64_t *, void **, void **);
+static void (*tgt_taskgraph_emit_target_update)(void *, int64_t, int32_t,
+ void **, void **, int64_t *,
+ int64_t *, void **, void **);
+static size_t (*tgt_taskgraph_end)(void *);
+static int (*tgt_taskgraph_finalize)(void *);
+static void (*tgt_taskgraph_begin_replay)(void *);
+static int (*tgt_taskgraph_replay)(void *);
+static void (*tgt_taskgraph_destroy)(void *);
+
// Deep-copies a kernel-arguments struct and the arrays it points at into a
// caller-provided block, sizing that block when the destination is null (see
// offload/include/omptarget.h).
@@ -70,6 +99,25 @@ static void (*tgt_taskgraph_dup_data_args)(void *, int32_t, void ***, void ***,
int64_t **, int64_t **, void ***,
void ***, size_t *);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+// True iff the full taskgraph handoff ABI resolved. Checked once before
+// streaming a graph; if false, libomp keeps the host-only path.
+static bool __kmp_tgt_taskgraph_available() {
+ return tgt_taskgraph_start && tgt_taskgraph_start_parallel &&
+ tgt_taskgraph_end_parallel && tgt_taskgraph_start_sequential &&
+ tgt_taskgraph_end_sequential && tgt_taskgraph_start_exclusive &&
+ tgt_taskgraph_end_exclusive && tgt_taskgraph_emit_target &&
+ tgt_taskgraph_emit_target_enter_data &&
+ tgt_taskgraph_emit_target_exit_data &&
+ tgt_taskgraph_emit_target_update && tgt_taskgraph_end &&
+ tgt_taskgraph_finalize && tgt_taskgraph_begin_replay &&
+ tgt_taskgraph_replay && tgt_taskgraph_destroy &&
+ tgt_taskgraph_dup_kernel_args && tgt_taskgraph_dup_data_args &&
+ tgt_taskgraph_start_irreducible && tgt_taskgraph_end_irreducible &&
+ tgt_taskgraph_emit_edge;
+}
+#endif
+
void __kmp_init_target_task() {
*(void **)(&tgt_target_nowait_query) = KMP_DLSYM("__tgt_target_nowait_query");
*(void **)(&tgt_target_kernel) = KMP_DLSYM("__tgt_target_kernel");
@@ -85,6 +133,38 @@ void __kmp_init_target_task() {
KMP_DLSYM("__tgt_target_data_end_nowait_mapper");
*(void **)(&tgt_target_data_update_nowait_mapper) =
KMP_DLSYM("__tgt_target_data_update_nowait_mapper");
+ *(void **)(&tgt_taskgraph_start) = KMP_DLSYM("__tgt_taskgraph_start");
+ *(void **)(&tgt_taskgraph_start_parallel) =
+ KMP_DLSYM("__tgt_taskgraph_start_parallel");
+ *(void **)(&tgt_taskgraph_end_parallel) =
+ KMP_DLSYM("__tgt_taskgraph_end_parallel");
+ *(void **)(&tgt_taskgraph_start_sequential) =
+ KMP_DLSYM("__tgt_taskgraph_start_sequential");
+ *(void **)(&tgt_taskgraph_end_sequential) =
+ KMP_DLSYM("__tgt_taskgraph_end_sequential");
+ *(void **)(&tgt_taskgraph_start_exclusive) =
+ KMP_DLSYM("__tgt_taskgraph_start_exclusive");
+ *(void **)(&tgt_taskgraph_end_exclusive) =
+ KMP_DLSYM("__tgt_taskgraph_end_exclusive");
+ *(void **)(&tgt_taskgraph_start_irreducible) =
+ KMP_DLSYM("__tgt_taskgraph_start_irreducible");
+ *(void **)(&tgt_taskgraph_end_irreducible) =
+ KMP_DLSYM("__tgt_taskgraph_end_irreducible");
+ *(void **)(&tgt_taskgraph_emit_edge) = KMP_DLSYM("__tgt_taskgraph_emit_edge");
+ *(void **)(&tgt_taskgraph_emit_target) =
+ KMP_DLSYM("__tgt_taskgraph_emit_target");
+ *(void **)(&tgt_taskgraph_emit_target_enter_data) =
+ KMP_DLSYM("__tgt_taskgraph_emit_target_enter_data");
+ *(void **)(&tgt_taskgraph_emit_target_exit_data) =
+ KMP_DLSYM("__tgt_taskgraph_emit_target_exit_data");
+ *(void **)(&tgt_taskgraph_emit_target_update) =
+ KMP_DLSYM("__tgt_taskgraph_emit_target_update");
+ *(void **)(&tgt_taskgraph_end) = KMP_DLSYM("__tgt_taskgraph_end");
+ *(void **)(&tgt_taskgraph_finalize) = KMP_DLSYM("__tgt_taskgraph_finalize");
+ *(void **)(&tgt_taskgraph_begin_replay) =
+ KMP_DLSYM("__tgt_taskgraph_begin_replay");
+ *(void **)(&tgt_taskgraph_replay) = KMP_DLSYM("__tgt_taskgraph_replay");
+ *(void **)(&tgt_taskgraph_destroy) = KMP_DLSYM("__tgt_taskgraph_destroy");
*(void **)(&tgt_taskgraph_dup_kernel_args) =
KMP_DLSYM("__tgt_taskgraph_dup_kernel_args");
*(void **)(&tgt_taskgraph_dup_data_args) =
@@ -2236,11 +2316,19 @@ __kmp_taskgraph_irreducible_order(kmp_info_t *thread,
return order;
}
+// The type REGION is replayed as. A region holding a libomptarget subgraph is
+// one step of the replay, however much it holds, and so is flattened like the
+// target construct it may well be.
+static enum kmp_taskgraph_region_type
+__kmp_taskgraph_exec_type(kmp_taskgraph_region_t *region) {
+ return region->tgt_graph ? TASKGRAPH_REGION_TARGET : region->type;
+}
+
// The number of exit descrs REGION resolves to, i.e. the length of the pred
// list __kmp_build_exec_descrs_1 returns for it. A dependency on REGION has to
// name every one of them.
static kmp_int32 __kmp_count_region_exits(kmp_taskgraph_region_t *region) {
- switch (region->type) {
+ switch (__kmp_taskgraph_exec_type(region)) {
case TASKGRAPH_REGION_NODE:
case TASKGRAPH_REGION_WAIT:
case TASKGRAPH_REGION_ENTRY:
@@ -2279,7 +2367,7 @@ static kmp_int32 __kmp_count_region_exits(kmp_taskgraph_region_t *region) {
kmp_int32 __kmp_count_exec_descrs_1(kmp_taskgraph_region_t *region,
kmp_int32 npreds, kmp_int32 &successors,
kmp_int32 &descrs) {
- switch (region->type) {
+ switch (__kmp_taskgraph_exec_type(region)) {
case TASKGRAPH_REGION_NODE:
case TASKGRAPH_REGION_WAIT:
case TASKGRAPH_REGION_ENTRY:
@@ -2533,7 +2621,7 @@ kmp_taskgraph_exec_descr_list_t *__kmp_build_exec_descrs_1(
walk = walk->next;
} while (walk != pred);
};
- switch (region->type) {
+ switch (__kmp_taskgraph_exec_type(region)) {
case TASKGRAPH_REGION_NODE:
case TASKGRAPH_REGION_WAIT:
case TASKGRAPH_REGION_ENTRY:
@@ -2711,7 +2799,7 @@ __kmp_build_exec_descrs(kmp_info_t *thread, kmp_taskgraph_region_t *region,
return exec_descr_arr;
}
-/// Reset, reparent and regroup the recorded task TASK and re-invoke it.
+/// Reset, reparent and regroup the recorded task of NODE and re-invoke it.
static void __kmp_omp_tg_task(kmp_int32 gtid, kmp_taskgraph_node_t *node,
kmp_taskgroup_t *taskgroup,
@@ -2765,8 +2853,9 @@ void *__kmp_task_reduction_init(int gtid, int num, T *data);
/// host addresses of mappings that originate on the stack, and refresh its
/// by-value firstprivates. What the helper is handed depends on the region
/// type: the kernel-arguments blob for a kernel construct, the map arrays for
-/// a data construct. The mask it returns is not needed: the host path
-/// re-issues from the refreshed arguments directly.
+/// a data construct. The mask it returns is not needed: the construct is
+/// re-issued from the refreshed arguments directly, here or in a libomptarget
+/// subgraph.
static void __kmp_taskgraph_relocate_target(kmp_taskgraph_target_node_t *target,
void *taskgraph_args) {
switch (target->kind) {
@@ -2831,6 +2920,73 @@ static void __kmp_taskgraph_exec_target(kmp_info_t *thread,
break;
}
}
+
+/// Re-issue every target construct of REGION, a subtree that holds nothing
+/// else, one at a time in an order its dependences allow.
+static void __kmp_taskgraph_exec_targets(kmp_info_t *thread,
+ kmp_taskgraph_region_t *region) {
+ switch (region->type) {
+ KMP_TASKGRAPH_REGION_TARGET_CASES:
+ __kmp_taskgraph_exec_target(thread, region);
+ return;
+ case TASKGRAPH_REGION_PARALLEL:
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++)
+ __kmp_taskgraph_exec_targets(thread, region->inner.children[c]);
+ return;
+ case TASKGRAPH_REGION_IRREDUCIBLE: {
+ kmp_taskgraph_region_t **order =
+ __kmp_taskgraph_irreducible_order(thread, region);
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++)
+ __kmp_taskgraph_exec_targets(thread, order[c]);
+ __kmp_fast_free(thread, order);
+ return;
+ }
+ default:
+ KMP_DEBUG_ASSERT(false && "host work in a target subgraph");
+ return;
+ }
+}
+
+/// Run the libomptarget subgraph REGION holds, and wait for it.
+///
+/// A subgraph libomptarget declines to run on this replay -- for a mapping
+/// state its plan cannot express, say -- has issued nothing, so its target
+/// constructs are re-issued here instead, as they would be had no plugin
+/// claimed it.
+static void __kmp_taskgraph_run_subgraph(kmp_info_t *thread,
+ kmp_taskgraph_region_t *region) {
+ KMP_ASSERT(tgt_taskgraph_replay);
+ if (tgt_taskgraph_replay(region->tgt_graph) == 0 /* OFFLOAD_SUCCESS */)
+ return;
+ if (__kmp_taskgraph_trace())
+ fprintf(stderr, "Taskgraph subgraph %p replay declined by libomptarget\n",
+ region->tgt_graph);
+ __kmp_taskgraph_exec_targets(thread, region);
+}
+
+/// Begin a replay of every libomptarget subgraph in REGION, once the recorded
+/// nodes have been relocated: this is when each one takes the values of its
+/// firstprivates.
+static void __kmp_taskgraph_begin_subgraphs(kmp_taskgraph_region_t *region) {
+ if (region->tgt_graph) {
+ KMP_ASSERT(tgt_taskgraph_begin_replay);
+ tgt_taskgraph_begin_replay(region->tgt_graph);
+ return;
+ }
+ switch (region->type) {
+ case TASKGRAPH_REGION_PARALLEL:
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ case TASKGRAPH_REGION_IRREDUCIBLE:
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++)
+ __kmp_taskgraph_begin_subgraphs(region->inner.children[c]);
+ break;
+ default:
+ break;
+ }
+}
#endif
// Decrement DESCR's predecessor count; if it reaches zero, fire it. TASKGROUP
@@ -2842,7 +2998,7 @@ static void __kmp_taskgraph_exec_descr_start(kmp_int32 gtid, kmp_info_t *thread,
if (npredecessors > 0)
return;
- switch (descr->region->type) {
+ switch (__kmp_taskgraph_exec_type(descr->region)) {
case TASKGRAPH_REGION_NODE: {
// Spawn the task; its successors are triggered later, on completion, by
// __kmp_taskgraph_exec_descr_finish.
@@ -2883,16 +3039,22 @@ static void __kmp_taskgraph_exec_descr_start(kmp_int32 gtid, kmp_info_t *thread,
break;
// clang-format off
KMP_TASKGRAPH_REGION_TARGET_CASES: {
- // Replay a recorded target construct inline. The libomptarget entry
- // points used here are synchronous, so the successors are released as soon
- // as the call returns -- as for a WAIT region, rather than from the
- // task-completion hook a NODE region relies on.
+ // Replay a recorded target construct, or a subgraph of them, inline. The
+ // libomptarget entry points used here are synchronous, so the successors
+ // are released as soon as the call returns -- as for a WAIT region, rather
+ // than from the task-completion hook a NODE region relies on. A subgraph
+ // takes no mutexes: a target construct that needs any is never put in one.
#if ENABLE_LIBOMPTARGET
- if (descr->region->mutexset)
- __kmp_taskgraph_acquire_locks(gtid, descr->region->owner, descr->region);
- __kmp_taskgraph_exec_target(thread, descr->region);
- if (descr->region->mutexset)
- __kmp_taskgraph_release_locks(gtid, descr->region->owner, descr->region);
+ kmp_taskgraph_region_t *region = descr->region;
+ if (region->tgt_graph) {
+ __kmp_taskgraph_run_subgraph(thread, region);
+ } else {
+ if (region->mutexset)
+ __kmp_taskgraph_acquire_locks(gtid, region->owner, region);
+ __kmp_taskgraph_exec_target(thread, region);
+ if (region->mutexset)
+ __kmp_taskgraph_release_locks(gtid, region->owner, region);
+ }
for (kmp_taskgraph_exec_descr_elem_t *s = descr->successors; s; s = s->next)
__kmp_taskgraph_exec_descr_start(gtid, thread, s->exec_descr, taskgroup);
#else
@@ -6284,6 +6446,7 @@ static void __kmp_taskgraph_reset(kmp_taskgraph_record_t *rec, kmp_int32 gtid,
rec->exec_descrs = nullptr;
rec->num_exec_descrs = 0;
rec->taskgraph_args = nullptr;
+ rec->num_tgt_subgraphs = 0;
rec->next = nullptr;
}
@@ -6322,6 +6485,16 @@ __kmp_taskgraph_free_region_metadata(kmp_info_t *thread,
if (region->mutexset) {
__kmp_fast_free(thread, region->mutexset);
}
+#if ENABLE_LIBOMPTARGET
+ // libomptarget references (does not copy) the target payloads of the nodes
+ // in a subgraph, so the subgraph is torn down before __kmp_taskgraph_free
+ // frees them.
+ if (region->tgt_graph) {
+ KMP_ASSERT(tgt_taskgraph_destroy);
+ tgt_taskgraph_destroy(region->tgt_graph);
+ region->tgt_graph = nullptr;
+ }
+#endif
switch (region->type) {
case TASKGRAPH_REGION_ENTRY:
case TASKGRAPH_REGION_EXIT:
@@ -6638,6 +6811,10 @@ void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
// different from the addresses seen at record time.
record->taskgraph_args = args;
__kmp_taskgraph_relocate_nodes(record);
+#if ENABLE_LIBOMPTARGET
+ if (record->num_tgt_subgraphs)
+ __kmp_taskgraph_begin_subgraphs(record->root);
+#endif
__kmp_replay_taskgraph(gtid, current_taskdata, record, graph_id, taskgroup);
__kmpc_end_taskgroup(loc_ref, gtid);
__kmp_taskgraph_end_replay(gtid, record);
@@ -6890,6 +7067,235 @@ static void __kmp_taskgraph_record_target_data(
target->u.data.arg_mappers = dup_arg_mappers;
}
+// --------------------------------------------------------------------------
+// Taskgraph libomptarget support (transmission of target subgraphs)
+// --------------------------------------------------------------------------
+
+// True iff REGION is a subtree libomptarget can run on its own: one that holds
+// target constructs and nothing else. A target construct that has to exclude
+// others through mutexinoutset is left out, as is a task reduction, since
+// libomp is what implements both.
+static bool __kmp_taskgraph_subtree_is_target(kmp_taskgraph_region_t *region) {
+ switch (region->type) {
+ KMP_TASKGRAPH_REGION_TARGET_CASES:
+ return !region->mutexset;
+ case TASKGRAPH_REGION_PARALLEL:
+ if (region->reduce_input)
+ return false;
+ KMP_FALLTHROUGH();
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ case TASKGRAPH_REGION_IRREDUCIBLE:
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++)
+ if (!__kmp_taskgraph_subtree_is_target(region->inner.children[c]))
+ return false;
+ return region->inner.num_children > 0;
+ default:
+ return false;
+ }
+}
+
+// State threaded through the preorder emit walk.
+struct kmp_tg_emit_ctx {
+ void *graph;
+};
+
+// Emit one target / target-data leaf to libomptarget.
+static void __kmp_taskgraph_emit_target_leaf(kmp_tg_emit_ctx *ctx,
+ kmp_taskgraph_region_t *region) {
+ void *graph = ctx->graph;
+ kmp_taskgraph_target_node_t *t = region->task.node->target;
+ KMP_DEBUG_ASSERT(t);
+ switch (t->kind) {
+ case TASKGRAPH_REGION_TARGET:
+ tgt_taskgraph_emit_target(graph, t->device_id, t->u.kernel.num_teams,
+ t->u.kernel.thread_limit, t->u.kernel.host_ptr,
+ t->u.kernel.kernel_args);
+ break;
+ case TASKGRAPH_REGION_TARGET_ENTER_DATA:
+ tgt_taskgraph_emit_target_enter_data(
+ graph, t->device_id, t->u.data.arg_num, t->u.data.args_base,
+ t->u.data.args, (int64_t *)t->u.data.arg_sizes,
+ (int64_t *)t->u.data.arg_types, t->u.data.arg_names,
+ t->u.data.arg_mappers);
+ break;
+ case TASKGRAPH_REGION_TARGET_EXIT_DATA:
+ tgt_taskgraph_emit_target_exit_data(
+ graph, t->device_id, t->u.data.arg_num, t->u.data.args_base,
+ t->u.data.args, (int64_t *)t->u.data.arg_sizes,
+ (int64_t *)t->u.data.arg_types, t->u.data.arg_names,
+ t->u.data.arg_mappers);
+ break;
+ case TASKGRAPH_REGION_TARGET_UPDATE:
+ tgt_taskgraph_emit_target_update(
+ graph, t->device_id, t->u.data.arg_num, t->u.data.args_base,
+ t->u.data.args, (int64_t *)t->u.data.arg_sizes,
+ (int64_t *)t->u.data.arg_types, t->u.data.arg_names,
+ t->u.data.arg_mappers);
+ break;
+ default:
+ KMP_DEBUG_ASSERT(false && "non-target kind in target leaf emit");
+ }
+}
+
+// Count or emit the explicit dependence edges of an irreducible region,
+// addressing children by emission ordinal (their timestamp, pre-assigned by the
+// caller). Returns the edge count so the caller can pass it to
+// start_irreducible before re-emitting.
+static kmp_int32 __kmp_taskgraph_irr_edges(kmp_tg_emit_ctx *ctx,
+ kmp_taskgraph_region_t *region,
+ bool emit) {
+ void *graph = ctx->graph;
+ kmp_int32 count = 0;
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+ kmp_taskgraph_region_t *child = region->inner.children[c];
+ for (kmp_taskgraph_region_dep_t *dep = child->successors; dep;
+ dep = dep->next)
+ if (dep->region->parent == region) {
+ if (emit)
+ tgt_taskgraph_emit_edge(graph, c, dep->region->timestamp);
+ count++;
+ }
+ }
+ return count;
+}
+
+// Preorder traversal of a target-only subtree of the processed region tree,
+// streaming its structure and leaves to libomptarget.
+static void __kmp_taskgraph_emit_region(kmp_tg_emit_ctx *ctx,
+ kmp_taskgraph_region_t *region) {
+ void *graph = ctx->graph;
+ switch (region->type) {
+ KMP_TASKGRAPH_REGION_TARGET_CASES:
+ __kmp_taskgraph_emit_target_leaf(ctx, region);
+ return;
+ case TASKGRAPH_REGION_PARALLEL:
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_EXCLUSIVE: {
+ void (*open)(void *, int32_t) = nullptr;
+ void (*close)(void *) = nullptr;
+ switch (region->type) {
+ case TASKGRAPH_REGION_PARALLEL:
+ open = tgt_taskgraph_start_parallel;
+ close = tgt_taskgraph_end_parallel;
+ break;
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ open = tgt_taskgraph_start_exclusive;
+ close = tgt_taskgraph_end_exclusive;
+ break;
+ default:
+ open = tgt_taskgraph_start_sequential;
+ close = tgt_taskgraph_end_sequential;
+ break;
+ }
+ open(graph, region->inner.num_children);
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++)
+ __kmp_taskgraph_emit_region(ctx, region->inner.children[c]);
+ close(graph);
+ return;
+ }
+ case TASKGRAPH_REGION_IRREDUCIBLE: {
+ // Children are numbered by their timestamp field so explicit edges below
+ // can reference them by index into the children array.
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++)
+ region->inner.children[c]->timestamp = c;
+ // The children of an irreducible region are transmitted before the edges.
+ kmp_int32 edge_count =
+ __kmp_taskgraph_irr_edges(ctx, region, /*emit=*/false);
+ tgt_taskgraph_start_irreducible(graph, region->inner.num_children,
+ edge_count);
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++)
+ __kmp_taskgraph_emit_region(ctx, region->inner.children[c]);
+ __kmp_taskgraph_irr_edges(ctx, region, /*emit=*/true);
+ tgt_taskgraph_end_irreducible(graph);
+ return;
+ }
+ default:
+ assert(false && "host work in a target subgraph");
+ }
+}
+
+// Offer the target-only subtree REGION of REC to libomptarget as a subgraph,
+// and keep the subgraph's handle on REGION if a plugin claims it.
+static void __kmp_taskgraph_transmit_subgraph(kmp_taskgraph_record_t *rec,
+ kmp_taskgraph_region_t *region) {
+ if (!__kmp_tgt_taskgraph_available()) {
+ if (__kmp_taskgraph_trace())
+ fprintf(stderr,
+ "Taskgraph %p has targets but the libomptarget handoff ABI is "
+ "unavailable; keeping host-only path\n",
+ rec);
+ return;
+ }
+
+ // Two-pass, single-block construction: the first pass streams the subgraph
+ // with ByteSize 0 so libomptarget only measures the size it needs; the
+ // identical second pass hands that size back so it can allocate one
+ // exactly-sized block that does not move during construction.
+
+ // Graph-level device id is the default device; per-node device ids are
+ // threaded through each emit call for later multi-device grouping.
+ void *graph = tgt_taskgraph_start(/*device_id=*/-1, rec->graph_id,
+ /*ByteSize=*/0);
+ if (!graph)
+ return;
+ kmp_tg_emit_ctx ctx = {graph};
+ __kmp_taskgraph_emit_region(&ctx, region);
+ size_t byte_size = tgt_taskgraph_end(graph);
+
+ graph = tgt_taskgraph_start(/*device_id=*/-1, rec->graph_id, byte_size);
+ if (!graph)
+ return;
+ ctx.graph = graph;
+ __kmp_taskgraph_emit_region(&ctx, region);
+ tgt_taskgraph_end(graph);
+
+ // If the default device's plugin will not take the subgraph, its target
+ // constructs are replayed one at a time like any others.
+ if (tgt_taskgraph_finalize(graph) != 0) {
+ tgt_taskgraph_destroy(graph);
+ if (__kmp_taskgraph_trace())
+ fprintf(stderr,
+ "Taskgraph %p subgraph at region %p was declined by the device "
+ "plugin\n",
+ rec, region);
+ return;
+ }
+
+ region->tgt_graph = graph;
+ rec->num_tgt_subgraphs++;
+ if (__kmp_taskgraph_trace())
+ fprintf(stderr,
+ "Transmitted taskgraph %p subgraph at region %p to libomptarget "
+ "(handle %p, %zu B)\n",
+ rec, region, graph, byte_size);
+}
+
+// Offer each maximal target-only subtree of REGION to libomptarget.
+static void __kmp_taskgraph_transmit_subtrees(kmp_taskgraph_record_t *rec,
+ kmp_taskgraph_region_t *region) {
+ if (__kmp_taskgraph_subtree_is_target(region)) {
+ __kmp_taskgraph_transmit_subgraph(rec, region);
+ return;
+ }
+ switch (region->type) {
+ case TASKGRAPH_REGION_PARALLEL:
+ case TASKGRAPH_REGION_SEQUENTIAL:
+ case TASKGRAPH_REGION_EXCLUSIVE:
+ case TASKGRAPH_REGION_IRREDUCIBLE:
+ for (kmp_int32 c = 0; c < region->inner.num_children; c++)
+ __kmp_taskgraph_transmit_subtrees(rec, region->inner.children[c]);
+ return;
+ default:
+ return;
+ }
+}
+
+void __kmp_taskgraph_transmit_to_target(kmp_int32 gtid,
+ kmp_taskgraph_record_t *rec) {
+ if (rec->root)
+ __kmp_taskgraph_transmit_subtrees(rec, rec->root);
+}
#endif
// Entry points for target constructs that should be recorded into a taskgraph,
>From 3dfbad1b8489e32dade3496d65725ad4ce4845f7 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 18 Sep 2026 08:15:26 -0500
Subject: [PATCH 20/29] [Offload] Add a data plan for recorded taskgraphs
This patch adds infrastructure for tracking the data requirements of a
finalized target taskgraph. This covers several distinct, but related
things.
First is the observation that it's useful to aggregate all the data
mappings that the target regions within a taskgraph may use. Different
target constructs may use overlapping regions, so we build a set of
unified ranges for mapped host data.
Second, some given target needs to pick subranges from those ranges,
and sometimes needs to perform associated actions, and sometimes not.
Third, each target leaf may perform several operations (copies-in,
attaches, launch, copies-out) referencing those arguments.
Fourth, between one replay and the next, changes can take place to the
mapping state in several ways. Non-saved firstprivate variables can
change their values, the stack frame of the replayed construct can move,
or target addresses associated with host data can appear, disappear
or move.
The next patch adds dumping for the plan representation which may be
helpful for understanding this patch.
The patch tries to reconcile the handling of reference counts for mapped
host data (i.e. on non-unified memory systems) with the needs of GPU
graph APIs which want to dispatch the whole graph for execution at once,
hence must bypass those reference counts to some degree. For OpenMP,
we typically only want to perform copy-in operations when the reference
count for the mapping transitions from 0->1, and copy-outs when it
transitions 1->0. The model we use is that copies are always emitted as
part of the graph, but are enabled or disabled depending on the current
mapping state at replay launch.
There is a static portion to the analysis: refcount changes are simulated
by a walk over the graph, and copies are only enabled at "perimeter"
nodes: those where the data is owned by the graph (not some enclosing
"enter data" construct for example) and not provably with refcount > 0.
There is also code to detect when a host region is owned by an in-progress
taskgraph replay, and some other thread tries to create another mapping
from the same data. That's a race condition, and we want to avoid
surprises where multiple copies of the host data exist on the target
at once.
New tests have been added, but some are XFAILed until an offload plugin
claims the graph.
Assisted-By: Claude
---
offload/include/OpenMP/Mapping.h | 76 +
offload/include/OpenMP/TaskGraphPlan.h | 107 +
offload/include/Shared/TaskGraph.h | 648 ++++++
offload/include/omptarget.h | 69 +
offload/libompaccsupport/Mapping.cpp | 168 +-
offload/libompaccsupport/PluginManager.cpp | 1 +
offload/libomptarget/CMakeLists.txt | 1 +
offload/libomptarget/OpenMP/API.cpp | 15 +
offload/libomptarget/OpenMP/TaskGraphPlan.cpp | 1751 +++++++++++++++++
offload/libomptarget/interface.cpp | 88 +-
offload/libomptarget/omptarget.cpp | 2 +
.../common/include/PluginInterface.h | 50 +
.../common/src/PluginInterface.cpp | 40 +
.../taskgraph_claim_attach_demotion.c | 83 +
.../offloading/taskgraph_claim_enter_data.c | 59 +
.../taskgraph_claim_finalize_partial.c | 77 +
.../test/offloading/taskgraph_claim_lookup.c | 58 +
.../taskgraph_claim_overlapping_graphs.c | 64 +
.../offloading/taskgraph_claim_relocated.c | 68 +
.../test/offloading/taskgraph_decline_info.c | 81 +
.../taskgraph_enter_data_struct_member.c | 56 +
.../taskgraph_enter_data_unbalanced.c | 73 +
.../offloading/taskgraph_exit_data_borrowed.c | 82 +
.../offloading/taskgraph_firstprivate_inner.c | 91 +
.../taskgraph_firstprivate_replay_start.c | 57 +
.../taskgraph_firstprivate_restage.c | 89 +
.../taskgraph_firstprivate_scalar.c | 102 +
.../taskgraph_firstprivate_scalar_reparam.c | 50 +
.../offloading/taskgraph_lambda_capture.cpp | 63 +
.../taskgraph_map_always_borrowed.c | 74 +
.../test/offloading/taskgraph_map_attach.c | 106 +
offload/test/offloading/taskgraph_map_close.c | 73 +
offload/test/offloading/taskgraph_map_drift.c | 115 ++
.../taskgraph_map_dynamic_bound_saved.c | 112 ++
.../offloading/taskgraph_map_partial_borrow.c | 81 +
.../test/offloading/taskgraph_map_private.c | 122 ++
.../taskgraph_map_relocated_mixed.c | 115 ++
.../taskgraph_map_relocated_stack.c | 90 +
.../offloading/taskgraph_map_struct_member.c | 62 +
.../offloading/taskgraph_present_absent.c | 71 +
.../offloading/taskgraph_present_replay.c | 53 +
.../offloading/taskgraph_target_decline.c | 61 +
.../taskgraph_unordered_copy_decline.c | 60 +
.../taskgraph_unordered_copy_replay.c | 81 +
.../offloading/taskgraph_unordered_shared.c | 73 +
45 files changed, 5489 insertions(+), 29 deletions(-)
create mode 100644 offload/include/OpenMP/TaskGraphPlan.h
create mode 100644 offload/libomptarget/OpenMP/TaskGraphPlan.cpp
create mode 100644 offload/test/offloading/taskgraph_claim_attach_demotion.c
create mode 100644 offload/test/offloading/taskgraph_claim_enter_data.c
create mode 100644 offload/test/offloading/taskgraph_claim_finalize_partial.c
create mode 100644 offload/test/offloading/taskgraph_claim_lookup.c
create mode 100644 offload/test/offloading/taskgraph_claim_overlapping_graphs.c
create mode 100644 offload/test/offloading/taskgraph_claim_relocated.c
create mode 100644 offload/test/offloading/taskgraph_decline_info.c
create mode 100644 offload/test/offloading/taskgraph_enter_data_struct_member.c
create mode 100644 offload/test/offloading/taskgraph_enter_data_unbalanced.c
create mode 100644 offload/test/offloading/taskgraph_exit_data_borrowed.c
create mode 100644 offload/test/offloading/taskgraph_firstprivate_inner.c
create mode 100644 offload/test/offloading/taskgraph_firstprivate_replay_start.c
create mode 100644 offload/test/offloading/taskgraph_firstprivate_restage.c
create mode 100644 offload/test/offloading/taskgraph_firstprivate_scalar.c
create mode 100644 offload/test/offloading/taskgraph_firstprivate_scalar_reparam.c
create mode 100644 offload/test/offloading/taskgraph_lambda_capture.cpp
create mode 100644 offload/test/offloading/taskgraph_map_always_borrowed.c
create mode 100644 offload/test/offloading/taskgraph_map_attach.c
create mode 100644 offload/test/offloading/taskgraph_map_close.c
create mode 100644 offload/test/offloading/taskgraph_map_drift.c
create mode 100644 offload/test/offloading/taskgraph_map_dynamic_bound_saved.c
create mode 100644 offload/test/offloading/taskgraph_map_partial_borrow.c
create mode 100644 offload/test/offloading/taskgraph_map_private.c
create mode 100644 offload/test/offloading/taskgraph_map_relocated_mixed.c
create mode 100644 offload/test/offloading/taskgraph_map_relocated_stack.c
create mode 100644 offload/test/offloading/taskgraph_map_struct_member.c
create mode 100644 offload/test/offloading/taskgraph_present_absent.c
create mode 100644 offload/test/offloading/taskgraph_present_replay.c
create mode 100644 offload/test/offloading/taskgraph_target_decline.c
create mode 100644 offload/test/offloading/taskgraph_unordered_copy_decline.c
create mode 100644 offload/test/offloading/taskgraph_unordered_copy_replay.c
create mode 100644 offload/test/offloading/taskgraph_unordered_shared.c
diff --git a/offload/include/OpenMP/Mapping.h b/offload/include/OpenMP/Mapping.h
index 4b160684ddb327..f4deddf0bf5ff2 100644
--- a/offload/include/OpenMP/Mapping.h
+++ b/offload/include/OpenMP/Mapping.h
@@ -17,8 +17,11 @@
#include "Shared/EnvironmentVar.h"
#include "omptarget.h"
+#include <atomic>
#include <cstdint>
+#include <map>
#include <mutex>
+#include <set>
#include <string>
#include "llvm/ADT/SmallSet.h"
@@ -674,6 +677,19 @@ struct MappingInfoTy {
bool ReleaseHDTTMap = true, StateInfoTy *StateInfo = nullptr,
const ident_t *Loc = nullptr);
+ /// Publish device storage the caller has already obtained as the mapping of
+ /// \p Size bytes at \p HstPtrBegin, with a reference count of one on the
+ /// hold count if \p UseHoldRefCount and on the dynamic count otherwise, and
+ /// tell the plugin about it. The caller must have established, under the
+ /// same accessor, that nothing is mapped there. Returns the new entry,
+ /// unlocked, or null if the plugin refused it -- in which case the entry
+ /// stays in the map, as getTargetPointer has always left it.
+ HostDataToTargetTy *insertMapping(HDTTMapAccessorTy &HDTTMap,
+ void *HstPtrBase, void *HstPtrBegin,
+ int64_t Size, uintptr_t TgtAllocBegin,
+ uintptr_t TgtPtrBegin, bool UseHoldRefCount,
+ map_var_info_t HstPtrName);
+
/// Return the target pointer for \p HstPtrBegin in \p HDTTMap. The accessor
/// ensures exclusive access to the HDTT map.
void *getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
@@ -713,6 +729,16 @@ struct MappingInfoTy {
int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size);
int disassociatePtr(void *HstPtrBegin);
+ /// A counter of changes to the set of mappings for this device, bumped
+ /// whenever an entry is added or removed.
+ uint64_t getEpoch() const { return Epoch.load(std::memory_order_acquire); }
+
+ /// Bump the counter \p getEpoch reports. To be called with the map accessor
+ /// held, so the bump corresponds to the change it describes.
+ void notifyMappingsChanged() {
+ Epoch.fetch_add(1, std::memory_order_release);
+ }
+
/// Print information about the transfer from \p HstPtr to \p TgtPtr (or vice
/// versa if \p H2D is false). If there is an existing mapping, or if \p Entry
/// is set, the associated metadata will be printed as well.
@@ -721,8 +747,58 @@ struct MappingInfoTy {
MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr,
const ident_t *Loc = nullptr, const char *Name = nullptr);
+ /// \name Taskgraph claims
+ ///
+ /// A taskgraph replay that gives a self-contained range device storage of its
+ /// own makes no mapping-table entry for it. A claim records that host
+ /// memory for the duration of the replay, so that a lookup or a new mapping
+ /// over it can be warned about. Claims are diagnostics only: every answer
+ /// they affect is the one an unmapped region gets.
+ ///
+ /// Claims from different replays may overlap, so they are kept as pieces cut
+ /// at every claim boundary, each counting the claims covering it.
+ /// @{
+ bool hasTaskGraphClaims() const {
+ return NumTaskGraphClaims.load(std::memory_order_acquire) != 0;
+ }
+ void addTaskGraphClaim(const void *Begin, int64_t Size);
+ void removeTaskGraphClaim(const void *Begin, int64_t Size);
+
+ /// Whether \p Size bytes at \p Begin, or the one address for a size of zero,
+ /// overlap a claim. If so, warn once per piece that \p What is answered as
+ /// for unmapped memory.
+ bool warnIfClaimedByTaskGraph(const void *Begin, int64_t Size,
+ const char *What);
+
+ /// Warn, once per piece, that \p What is creating a mapping over a claim,
+ /// which makes a second device copy of the same bytes.
+ void warnMappingOverTaskGraphClaim(const void *Begin, int64_t Size,
+ const char *What);
+ /// @}
+
private:
DeviceTy &Device;
+ std::atomic<uint64_t> Epoch{0};
+
+ struct TaskGraphClaimTy {
+ uintptr_t Hi = 0;
+ uint32_t Claims = 0;
+ };
+ /// Pieces keyed by their begin. NumTaskGraphClaims mirrors the size so that
+ /// callers can skip the lock when nothing is claimed.
+ std::map<uintptr_t, TaskGraphClaimTy> TaskGraphClaims;
+ std::atomic<size_t> NumTaskGraphClaims{0};
+ std::mutex TaskGraphClaimsMtx;
+ /// The piece begins each warning has been given for, kept apart from the
+ /// pieces so that a range claimed on every replay is warned about once.
+ std::set<uintptr_t> TaskGraphLookupsWarned;
+ std::set<uintptr_t> TaskGraphMapsWarned;
+
+ /// Make a piece begin at \p Addr, if one spans it. Lock held.
+ void splitTaskGraphClaim(uintptr_t Addr);
+ /// The first piece reaching past \p Lo. Lock held.
+ std::map<uintptr_t, TaskGraphClaimTy>::iterator
+ firstTaskGraphClaimAfter(uintptr_t Lo);
};
#endif // OMPTARGET_OPENMP_MAPPING_H
diff --git a/offload/include/OpenMP/TaskGraphPlan.h b/offload/include/OpenMP/TaskGraphPlan.h
new file mode 100644
index 00000000000000..b5e44afbbf97df
--- /dev/null
+++ b/offload/include/OpenMP/TaskGraphPlan.h
@@ -0,0 +1,107 @@
+//===-- OpenMP/TaskGraphPlan.h - A taskgraph's mapped data ------- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Deciding where a recorded taskgraph's mapped data lives, and keeping that
+// decision current across replays. See TaskGraphDataPlanTy in
+// Shared/TaskGraph.h for the representation.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_OPENMP_TASKGRAPHPLAN_H
+#define OMPTARGET_OPENMP_TASKGRAPHPLAN_H
+
+#include "Shared/TaskGraph.h"
+
+#include "llvm/ADT/SmallVector.h"
+
+struct DeviceTy;
+
+namespace llvm {
+namespace omp {
+namespace target {
+
+/// Build \p Graph's plan: gather every leaf's mapped host ranges into disjoint
+/// ranges, place each argument in its range, and decide which maps perform the
+/// implicit transfers. Device addresses are left for the first replay to
+/// resolve. Call once, before the graph is offered to a plugin.
+///
+/// Returns false if the graph maps something the plan cannot express. The
+/// graph has not been claimed yet, so libomp then replays it on the host.
+bool buildTaskGraphPlan(DeviceTy &Device, TaskGraphTy &Graph);
+
+/// Begin a replay of the taskgraph \p Graph belongs to: take the initial
+/// values of its firstprivates from their variables, as they stand when the
+/// replay starts (OpenMP 6.0 [14.3]), however much later the graph runs.
+void beginTaskGraphReplay(TaskGraphTy &Graph);
+
+/// Resolve every one of \p Graph's ranges against the device's mapping table
+/// for a replay about to start, and report the argument entries that changed
+/// in \p Updates. \p Resync is set when the backend has to re-read the whole
+/// plan instead, because a replay it never saw may have changed it.
+///
+/// A range the graph brings into the device data environment and finds absent
+/// is mapped, so that a `target enter data` in a graph behaves as it does
+/// outside one, including leaving the mapping behind when the graph's maps do
+/// not balance. A self-contained range gets storage of its own instead. Every
+/// range is then held with a hold reference until the replay ends.
+///
+/// Only addresses, ownership and transfer enablement are re-derived; the
+/// grouping and extents are fixed, so a backend never has to change the shape
+/// of a lowered graph.
+///
+/// Must be paired with releaseTaskGraphMappings even when it returns false,
+/// since a partial acquisition still holds what it took; TaskGraphMappingsGuard
+/// does that. Returns false for a mapping state the plan cannot express, which
+/// declines this replay to the host.
+bool acquireTaskGraphMappings(DeviceTy &Device, TaskGraphTy &Graph,
+ SmallVectorImpl<TaskGraphArgUpdateTy> &Updates,
+ bool &Resync);
+
+/// Tell the plan that the updates the last acquisition produced have reached
+/// the backend, so the next one need only report what changes from here.
+void commitTaskGraphUpdates(TaskGraphTy &Graph);
+
+/// Drop the hold references acquireTaskGraphMappings took, and remove the
+/// mappings the graph created that its maps did not ask to outlive it.
+/// \p Replayed is false for a declined replay, which moved no data and so has
+/// no attached pointers to restore.
+void releaseTaskGraphMappings(DeviceTy &Device, TaskGraphTy &Graph,
+ bool Replayed = true);
+
+/// Releases a graph's mappings unless the acquisition reaches the point of
+/// handing the graph to the backend, which then releases them once the graph
+/// has run.
+class TaskGraphMappingsGuard {
+public:
+ TaskGraphMappingsGuard(DeviceTy &Device, TaskGraphTy &Graph)
+ : Device(Device), Graph(Graph) {}
+ ~TaskGraphMappingsGuard() {
+ if (Armed)
+ releaseTaskGraphMappings(Device, Graph, /*Replayed=*/false);
+ }
+
+ /// Hand the hold to the replay.
+ void release() { Armed = false; }
+
+ TaskGraphMappingsGuard(const TaskGraphMappingsGuard &) = delete;
+ TaskGraphMappingsGuard &operator=(const TaskGraphMappingsGuard &) = delete;
+
+private:
+ DeviceTy &Device;
+ TaskGraphTy &Graph;
+ bool Armed = true;
+};
+
+/// Release everything the plan holds for \p Graph, for teardown.
+void releaseTaskGraphPlan(DeviceTy &Device, TaskGraphTy &Graph);
+
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OMPTARGET_OPENMP_TASKGRAPHPLAN_H
diff --git a/offload/include/Shared/TaskGraph.h b/offload/include/Shared/TaskGraph.h
index fef340bb1dd879..ef4bd206670982 100644
--- a/offload/include/Shared/TaskGraph.h
+++ b/offload/include/Shared/TaskGraph.h
@@ -28,12 +28,15 @@
#include "omptarget.h"
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/AlignOf.h"
#include "llvm/Support/Casting.h"
#include <cassert>
#include <cstdint>
#include <cstdlib>
#include <new>
+#include <type_traits>
+#include <utility>
#include <vector>
namespace llvm {
@@ -67,6 +70,10 @@ enum class TGKind : uint8_t {
};
struct TaskGraphRegion;
+class TaskGraphArgBaseTy;
+struct TaskGraphOpTy;
+struct TaskGraphKernelParamTy;
+class TaskGraphPlanner;
/// Common base for every taskgraph element.
struct TaskGraphElement {
@@ -161,6 +168,183 @@ struct TaskGraphIrreducibleRegion : TaskGraphRegion {
}
};
+class TaskGraphRangeTy;
+
+/// What the plan does with one argument of one leaf.
+///
+/// Every leaf with map arrays has one entry per recorded argument, indexed like
+/// the map arrays. An entry names its range directly, with a range-relative
+/// offset, so it sees the range's current addresses without any update.
+class TaskGraphArgBaseTy {
+public:
+ enum class KindTy : uint8_t {
+ None,
+ Literal,
+ Mapped,
+ Attach,
+ NoTargetAttach,
+ FirstPrivate
+ };
+ KindTy getKind() const { return Kind; }
+
+protected:
+ explicit TaskGraphArgBaseTy(KindTy K) : Kind(K) {}
+
+private:
+ friend class TaskGraphPlanner;
+ friend class TaskGraphDataPlanTy;
+
+ KindTy Kind = KindTy::None;
+};
+
+class TaskGraphNoneArgTy : public TaskGraphArgBaseTy {
+public:
+ TaskGraphNoneArgTy() : TaskGraphArgBaseTy(KindTy::None) {}
+
+ static bool classof(const TaskGraphArgBaseTy *A) {
+ return A->getKind() == KindTy::None;
+ }
+};
+
+class TaskGraphLiteralArgTy : public TaskGraphArgBaseTy {
+public:
+ TaskGraphLiteralArgTy() : TaskGraphArgBaseTy(KindTy::Literal) {}
+ TaskGraphLiteralArgTy(uint64_t Val)
+ : TaskGraphArgBaseTy(KindTy::Literal), LiteralValue(Val) {}
+
+ /// Literal kernel argument. Refreshed by relocation.
+ uint64_t LiteralValue = 0;
+
+ static bool classof(const TaskGraphArgBaseTy *A) {
+ return A->getKind() == KindTy::Literal;
+ }
+};
+
+struct TaskGraphCopyTy {
+ // TRUE if this copy kind was implied or present in a clause
+ bool Requested = false;
+ // TRUE if a static walk over the graph at finalize time showed that this is
+ // the construct that should perform the copy (moving the ref count to or
+ // from zero). Only takes effect when the mapping is created by the graph,
+ // i.e. not for data that is already present on the device when a replay
+ // starts executing.
+ bool Perimeter = false;
+ // TRUE if this copy is enabled for a given replay.
+ bool Enabled = true;
+};
+
+class TaskGraphMappedArgTy : public TaskGraphArgBaseTy {
+public:
+ TaskGraphMappedArgTy() : TaskGraphArgBaseTy(KindTy::Mapped) {}
+
+ /// The argument lies in Range at Offset.
+ TaskGraphRangeTy *Range = nullptr;
+ uint64_t Offset = 0;
+ uint64_t Size = 0;
+
+ TaskGraphCopyTy In;
+ TaskGraphCopyTy Out;
+ bool OwnsBorrowedOut = false;
+ bool Always = false;
+
+ static bool classof(const TaskGraphArgBaseTy *A) {
+ return A->getKind() == KindTy::Mapped;
+ }
+};
+
+class TaskGraphAttachArgTy : public TaskGraphArgBaseTy {
+public:
+ TaskGraphAttachArgTy() : TaskGraphArgBaseTy(KindTy::Attach) {}
+ TaskGraphAttachArgTy(TaskGraphRangeTy *PtrRange, uint64_t Offset,
+ uint64_t Size)
+ : TaskGraphArgBaseTy(KindTy::Attach), PtrRange(PtrRange), Offset(Offset),
+ Size(Size) {}
+
+ /// A pointer attachment: the device copy of the pointer, at Offset in
+ /// PtrRange, is made to point at the device copy of the pointee. The bytes
+ /// written come from staging the plan fills each replay. The write is
+ /// unconditional, unlike outside a graph, because the device copy of the
+ /// pointer may have been made by a different replay.
+ TaskGraphRangeTy *PtrRange = nullptr;
+ uint64_t Offset = 0;
+ uint64_t Size = 0;
+
+ // Per-replay enabled flag.
+ bool Enabled = false;
+
+ TaskGraphRangeTy *PteeRange = nullptr;
+ int64_t PteeOffset = 0;
+
+ // Offset of attachment's bytes in AttachStaging and AttachHostBytes.
+ uint64_t StagingOffset = 0;
+
+ static bool classof(const TaskGraphArgBaseTy *A) {
+ return A->getKind() == KindTy::Attach;
+ }
+};
+
+// This is a no-op: keeping it as a distinct value (from e.g. "None") for
+// self-documentation and debug dumps, etc.
+
+class TaskGraphNoTargetAttachArgTy : public TaskGraphArgBaseTy {
+public:
+ TaskGraphNoTargetAttachArgTy() : TaskGraphArgBaseTy(KindTy::NoTargetAttach) {}
+
+ static bool classof(const TaskGraphArgBaseTy *A) {
+ return A->getKind() == KindTy::NoTargetAttach;
+ }
+};
+
+class TaskGraphFirstPrivateArgTy : public TaskGraphArgBaseTy {
+public:
+ TaskGraphFirstPrivateArgTy() : TaskGraphArgBaseTy(KindTy::FirstPrivate) {}
+
+ uint64_t Size = 0;
+
+ uint64_t StagingOffset = 0;
+ int64_t BaseOffset = 0;
+
+ // Each of these is per-replay.
+ bool Saved = false;
+ bool Staged = false;
+ bool InitPending = false;
+
+ static bool classof(const TaskGraphArgBaseTy *A) {
+ return A->getKind() == KindTy::FirstPrivate;
+ }
+};
+
+/// Container for any of the argument types above -- we want to be able to store
+/// them as values in an array, so we need to know the largest size. This
+/// allows us to do that.
+
+template <typename... ArgTys> class TaskGraphArgSlotBaseTy {
+ static_assert((std::is_trivially_copyable_v<ArgTys> && ...),
+ "non-trivial argument type");
+ llvm::AlignedCharArrayUnion<ArgTys...> Storage;
+
+public:
+ TaskGraphArgSlotBaseTy() { emplace<TaskGraphNoneArgTy>(); }
+ template <typename T, typename... As> T &emplace(As &&...Args) {
+ static_assert((std::is_same_v<T, ArgTys> || ...), "not a slot kind");
+ return *new (Storage.buffer) T(std::forward<As>(Args)...);
+ }
+ TaskGraphArgBaseTy &get() {
+ return *std::launder(
+ reinterpret_cast<TaskGraphArgBaseTy *>(Storage.buffer));
+ }
+ const TaskGraphArgBaseTy &get() const {
+ return *std::launder(
+ reinterpret_cast<const TaskGraphArgBaseTy *>(Storage.buffer));
+ }
+};
+
+using TaskGraphArgSlotTy =
+ TaskGraphArgSlotBaseTy<TaskGraphNoneArgTy, TaskGraphLiteralArgTy,
+ TaskGraphMappedArgTy, TaskGraphAttachArgTy,
+ TaskGraphNoTargetAttachArgTy,
+ TaskGraphFirstPrivateArgTy>;
+
//===----------------------------------------------------------------------===//
// Leaves (nodes)
//===----------------------------------------------------------------------===//
@@ -169,6 +353,12 @@ struct TaskGraphIrreducibleRegion : TaskGraphRegion {
struct TaskGraphNode : TaskGraphElement {
int64_t DeviceId = OFFLOAD_DEVICE_DEFAULT;
+ /// The leaf's share of the data plan, one entry per recorded map argument
+ /// and indexed like the map arrays; empty for a leaf with no map arrays.
+ inline llvm::ArrayRef<TaskGraphArgSlotTy> argPlan() const;
+ /// The operations the leaf performs, in order.
+ inline llvm::ArrayRef<TaskGraphOpTy> ops() const;
+
static bool classof(const TaskGraphElement *E) {
return E->getKind() >= TGKind::TargetNode &&
E->getKind() <= TGKind::TargetUpdateDataNode;
@@ -176,6 +366,15 @@ struct TaskGraphNode : TaskGraphElement {
protected:
explicit TaskGraphNode(TGKind K) : TaskGraphElement(K) {}
+
+private:
+ friend class TaskGraphPlanner;
+
+ // Set when the plan is built.
+ TaskGraphArgSlotTy *ArgPlan = nullptr;
+ uint32_t NumArgPlans = 0;
+ TaskGraphOpTy *OpPlan = nullptr;
+ uint32_t NumOps = 0;
};
/// A target kernel launch. KernelArgs is an opaque KernelArgsTy*, deep-copied
@@ -185,11 +384,20 @@ struct TaskGraphTargetNode : TaskGraphNode {
int32_t ThreadLimit = 0;
void *HostPtr = nullptr;
void *KernelArgs = nullptr;
+ /// The kernel's explicit parameters, in the kernel's order.
+ inline llvm::ArrayRef<TaskGraphKernelParamTy> kernelParams() const;
TaskGraphTargetNode() : TaskGraphNode(TGKind::TargetNode) {}
static bool classof(const TaskGraphElement *E) {
return E->getKind() == TGKind::TargetNode;
}
+
+private:
+ friend class TaskGraphPlanner;
+
+ // Set when the plan is built.
+ TaskGraphKernelParamTy *Params = nullptr;
+ uint32_t NumParams = 0;
};
/// Base for target enter/exit/update data ops. The specific op is encoded by
@@ -255,6 +463,441 @@ struct TaskGraphServicesTy {
void *(*resolveKernel)(int64_t DeviceId, void *HostPtr) = nullptr;
};
+//===----------------------------------------------------------------------===//
+// The data plan: where a graph's mapped data lives
+//
+// Built by libomptarget before the graph is offered to a plugin and kept
+// current across replays, so that every backend lowers the same decisions
+// about which host bytes have which device copy. Backends differ only in how
+// they perform the transfers the plan describes.
+//===----------------------------------------------------------------------===//
+
+/// One host range the graph maps, after the maps sharing host memory have been
+/// gathered into one. Gathering keeps each host byte to a single device copy.
+///
+/// The grouping and the extents are fixed when the graph is finalized; only the
+/// addresses are re-derived. A backend that lowered the graph onto a native
+/// graph object can update a node's parameters but cannot add or remove nodes,
+/// so the set of ranges has to be an invariant of the graph.
+///
+/// A range lives in TaskGraphDataPlanTy::Ranges for the graph's lifetime and
+/// never moves in memory, so argument entries and backend side tables may hold
+/// its address or index by its Id.
+class TaskGraphRangeTy {
+public:
+ /// The range's index in the plan's ranges(), which is in host-address order
+ /// when the plan is built. What a backend indexes a side table by.
+ uint32_t getId() const { return Id; }
+
+ /// The host range. The begin follows relocation; the extent is fixed.
+ void *getHostBegin() const { return HostBegin; }
+ uint64_t getSize() const { return Size; }
+ char *begin() const { return static_cast<char *>(HostBegin); }
+ char *end() const { return begin() + Size; }
+
+ /// The device address for this replay: the graph's own storage for a
+ /// self-contained range, and otherwise that of the mapping-table entry
+ /// covering the range. Null while the range is absent.
+ void *getDevicePtr() const { return DevicePtr; }
+
+private:
+ friend class TaskGraphPlanner;
+ friend class TaskGraphDataPlanTy;
+
+ uint32_t Id = 0;
+ void *HostBegin = nullptr;
+ uint64_t Size = 0;
+ void *DevicePtr = nullptr;
+
+ // Plan-builder state.
+
+ /// The graph owns the range outright: it creates the range, its maps balance,
+ /// and nothing outside the graph had it mapped when the graph was finalized.
+ /// Each replay gives such a range device storage of its own and no
+ /// mapping-table entry; a claim (see MappingInfoTy) records the storage for
+ /// diagnostics. A replay that finds the range mapped from outside borrows
+ /// that mapping instead, and one that finds it partly mapped declines to the
+ /// host. Fixed at finalize.
+ bool SelfContained = false;
+
+ /// This replay allocated DevicePtr as a self-contained range's storage, and
+ /// frees it when the replay ends.
+ bool StorageHeld = false;
+
+ /// The graph created this replay's mapping of the range, so the graph
+ /// performs the range's implicit transfers and removes the mapping again.
+ /// False means the range was already mapped when the replay began, and the
+ /// transfers and the lifetime belong to whoever mapped it, unless the graph
+ /// drops its last reference (DropsLast).
+ bool PlanInserted = false;
+
+ /// The range was already mapped when the replay began, and the reference the
+ /// graph's exit-data maps drop is that mapping's last, so the graph removes
+ /// the mapping and performs the copy out it implies.
+ bool DropsLast = false;
+
+ /// The graph's enter-data and exit-data maps' net effect on the range's
+ /// reference count: +1 for each enter and -1 for each exit, not counting a
+ /// struct member's map, which shares its struct's reference. See persists()
+ /// and releases().
+ int32_t RefDelta = 0;
+
+ /// Some leaf may bring the range into the device data environment: an
+ /// enter-data map or a `target` region's map. A range named only by
+ /// `target update` is never created by the graph; its transfers are switched
+ /// off while it is absent.
+ bool MayCreate = false;
+
+ /// Some map naming the range carried `present`.
+ bool RequiresPresent = false;
+
+ /// An exit-data map named the range with `delete`.
+ bool Deleted = false;
+
+ /// Some map naming the range carried `ompx_hold`, so the reference an
+ /// unbalanced graph leaves behind goes on the hold count. Unrelated to Held.
+ bool Holds = false;
+
+ /// Some map naming the range carried `close`, so under unified shared memory
+ /// the range gets device memory rather than using the host allocation. The
+ /// range is one device copy, so one map asking is enough.
+ bool Close = false;
+
+ /// The device uses the host memory directly this replay, under unified
+ /// shared memory or automatic zero-copy, so there is nothing to copy.
+ bool HostResident = false;
+
+ /// The graph holds a hold reference on the range's mapping until the replay
+ /// ends, so the mapping cannot be removed while the graph uses it.
+ bool Held = false;
+
+ /// The relocation that last moved the range, numbered by
+ /// TaskGraphDataPlanTy::Relocations, and the distance it moved. Used to
+ /// check that every argument in the range moved by the same distance.
+ uint64_t MovedInRelocation = 0;
+ int64_t MovedBy = 0;
+
+ /// A mapping the graph creates for the range outlives the replay, as one
+ /// made by an unbalanced `target enter data` does.
+ bool persists() const { return RefDelta > 0 && !Deleted; }
+ /// The replay ends one reference down on the mapping it found, as an
+ /// unbalanced `target exit data` does.
+ bool releases() const { return RefDelta < 0; }
+};
+
+llvm::ArrayRef<TaskGraphArgSlotTy> TaskGraphNode::argPlan() const {
+ return {ArgPlan, NumArgPlans};
+}
+
+/// One operation of a leaf. A leaf's operations run in the order ops() lists
+/// them, which for a target leaf is its copy-ins, its attachments and private
+/// initialisations, the kernel, and its copy-outs, and for a data leaf its
+/// copies in argument order, a to/from argument's copy-in first, followed by
+/// its attachments.
+///
+/// Every copy-like operation is listed whether or not the current replay
+/// performs it; see TaskGraphDataPlanTy::transfer.
+struct TaskGraphOpTy {
+ enum class KindTy : uint8_t {
+ /// Copy a Mapped argument from the host to the device.
+ CopyIn,
+ /// Copy a Mapped argument from the device to the host.
+ CopyOut,
+ /// Write an Attach argument's staged pointer into the device copy.
+ Attach,
+ /// Write a Private argument's staged initial value into its storage.
+ PrivateInit,
+ /// Launch the target leaf's kernel.
+ Launch,
+ };
+ KindTy Kind;
+ /// The argument entry performed, as an index into the leaf's argPlan().
+ /// Unused for Launch.
+ uint32_t Arg = 0;
+
+ bool isCopyLike() const { return Kind != KindTy::Launch; }
+};
+
+/// One explicit parameter of a target leaf's kernel.
+struct TaskGraphKernelParamTy {
+ enum class KindTy : uint8_t {
+ /// The device address of a Mapped argument, at Offset from its range's.
+ Mapped,
+ /// The address of a Private argument's storage, at Offset from it.
+ Private,
+ /// A Literal argument's value.
+ Literal,
+ /// The kernel launch environment, which the backend provides.
+ LaunchEnvironment,
+ };
+ KindTy Kind;
+ /// The recorded argument, as an index into the leaf's argPlan(). Unused
+ /// for LaunchEnvironment.
+ uint32_t Arg = 0;
+ /// The kernel is handed a base address, which for a struct member or a
+ /// pointer-and-pointee map lies before the mapped begin, so this can be
+ /// negative. The distance is a property of the map's shape and is fixed
+ /// for the graph.
+ int64_t Offset = 0;
+};
+
+/// Where a copy-like operation moves its bytes this replay.
+struct TaskGraphTransferTy {
+ /// Either end is null while it cannot be resolved, such as the device end
+ /// of a range absent this replay; the operation is then disabled.
+ void *Dst = nullptr;
+ const void *Src = nullptr;
+ /// The range the host end lies in, or null when the host end is the plan's
+ /// own staging. A backend that registers or pins host memory per range
+ /// finds the host end's buffer here.
+ const TaskGraphRangeTy *HostRange = nullptr;
+ uint64_t Size = 0;
+ bool ToDevice = true;
+ /// This replay performs the operation.
+ bool Enabled = false;
+};
+
+/// What changed about one argument entry since the last replay the backend
+/// took in, for a backend that has already lowered the graph.
+///
+/// Every operation and kernel parameter a backend lowers comes from one
+/// argument entry, so the entry is the unit changes are reported against: an
+/// attachment depends on the ranges of both its pointer and its pointee, but
+/// the plan resolves that and reports the attachment once. Values themselves
+/// are always read from the plan; an update only says which ones to re-read.
+///
+/// An update only repoints addresses, refreshes bytes the plan supplies and
+/// switches operations on or off; it never adds or removes an operation. A
+/// native-graph backend can therefore serve it with node parameter updates and
+/// node enablement. Because nothing is added later, a backend has to lower
+/// every copy-like operation whether or not the current replay performs it. A
+/// range named only by `target update` can be absent when the graph is built,
+/// with no device address and its transfers disabled, and mapped by the
+/// program before a later replay. A native-graph backend therefore points such
+/// an operand at scratch of its own, sized for the largest such transfer, and
+/// enables the node once the range resolves.
+struct TaskGraphArgUpdateTy {
+ enum ChangeTy : uint8_t {
+ /// The entry's device address moved: a Mapped entry's or an attachment's
+ /// pointer's range moved on the device.
+ DeviceAddressChanged = 1 << 0,
+ /// The entry's host address moved: relocation moved a Mapped entry's
+ /// range.
+ HostAddressChanged = 1 << 1,
+ /// The bytes the plan supplies for the entry changed: an attachment's or a
+ /// private's staged bytes, or a literal's value. Program memory a copy
+ /// reads when it runs is not such bytes.
+ DataChanged = 1 << 2,
+ /// Whether the replay performs the entry's copy-like operations changed.
+ EnabledChanged = 1 << 3,
+ /// Every kind of change, including any added later: what a backend has to
+ /// assume of every entry when it is told to resync.
+ AllChanged = 0xff,
+ };
+ /// The entry, as an index into TaskGraphDataPlanTy::Args.
+ uint32_t Arg = 0;
+ /// A mask of ChangeTy.
+ uint8_t Changes = 0;
+
+ bool has(ChangeTy Change) const { return Changes & Change; }
+};
+
+/// Where a graph's mapped data lives. libomptarget builds it and keeps it
+/// current; a backend reads it through the members below.
+class TaskGraphDataPlanTy {
+public:
+ /// The ranges, indexed by Id. They never overlap, and ranges that merely
+ /// abut are kept apart. Fixed in number when the plan is built, so a
+ /// backend may hold a range's address.
+ llvm::ArrayRef<TaskGraphRangeTy> ranges() const { return Ranges; }
+
+ /// Every leaf's argument entries, end to end, as an update names them.
+ size_t getNumArgs() const { return Args.size(); }
+ const TaskGraphArgSlotTy &getArg(uint32_t Index) const { return Args[Index]; }
+ /// \p A's index among the entries, which is what an update names it by.
+ uint32_t argIndex(const TaskGraphArgSlotTy &A) const {
+ return static_cast<uint32_t>(&A - Args.data());
+ }
+
+ /// The bytes every attachment writes and the host image of the private
+ /// block. Refilled in place each replay, at addresses fixed for the
+ /// graph's lifetime, so a backend may lower a copy from them once.
+ llvm::ArrayRef<char> attachStaging() const { return AttachStaging; }
+ llvm::ArrayRef<char> privateStaging() const { return PrivateStaging; }
+ /// Device storage for every private copy in the graph, allocated when the
+ /// plan is built and freed with it. Not in the mapping table.
+ void *getPrivateBlock() const { return PrivateBlock; }
+
+ /// Where \p Op, a copy-like operation of \p Leaf, moves its bytes this
+ /// replay. The size and direction are fixed for the graph.
+ TaskGraphTransferTy transfer(const TaskGraphNode &Leaf,
+ const TaskGraphOpTy &Op) const;
+
+ /// The value \p Leaf's kernel is handed for \p Param this replay, given the
+ /// backend's launch environment \p LaunchEnvironment.
+ uint64_t paramValue(const TaskGraphTargetNode &Leaf,
+ const TaskGraphKernelParamTy &Param,
+ void *LaunchEnvironment) const;
+
+ /// The last acquisition declined its replay for a reason it has reported
+ /// itself, so the caller need not.
+ bool declineReported() const { return DeclineReported; }
+
+private:
+ friend class TaskGraphPlanner;
+
+ const char *attachStagingFor(const TaskGraphAttachArgTy &A) const {
+ return AttachStaging.data() + A.StagingOffset;
+ }
+ const char *privateStagingFor(const TaskGraphFirstPrivateArgTy &F) const {
+ return PrivateStaging.data() + F.StagingOffset;
+ }
+ char *privateDevicePtr(const TaskGraphFirstPrivateArgTy &F) const {
+ return static_cast<char *>(PrivateBlock) + F.StagingOffset;
+ }
+
+ std::vector<TaskGraphRangeTy> Ranges;
+ /// Never resized after the plan is built, so pointers into it are stable.
+ std::vector<TaskGraphArgSlotTy> Args;
+ std::vector<char> AttachStaging;
+ /// What each attached pointer held on the host when the replay started, at
+ /// the same offsets as AttachStaging. A `from` copy of the range holding
+ /// the pointer brings the device value back, so the host value is restored
+ /// when the replay ends, as the runtime does at the end of a data region.
+ std::vector<char> AttachHostBytes;
+ std::vector<char> PrivateStaging;
+ /// What each private pointer initialisation's host object held when the
+ /// replay began, at the same offsets as PrivateStaging.
+ std::vector<char> PrivateHostBytes;
+ void *PrivateBlock = nullptr;
+ /// Every leaf's operations, end to end; each leaf's OpPlan spans its own.
+ std::vector<TaskGraphOpTy> Ops;
+ /// Every target leaf's kernel parameters, end to end.
+ std::vector<TaskGraphKernelParamTy> KernelParams;
+
+ /// The mapping-table epoch as of the plan's build or its last release of the
+ /// mappings. Only used to report that the program changed the mapping state
+ /// between replays; every replay resolves every range regardless.
+ uint64_t Epoch = 0;
+
+ /// The graph holds its mappings for a replay in flight.
+ bool MappingsHeld = false;
+
+ /// How many replays have re-keyed the ranges. Compared against a range's
+ /// MovedInRelocation; zero stamps no range.
+ uint64_t Relocations = 0;
+
+ /// The backend may not have adopted every change. A replay declined to the
+ /// host can change the plan without telling the backend, so the next replay
+ /// to reach it has the backend re-read everything. Set at the start of each
+ /// acquisition and cleared by commitTaskGraphUpdates.
+ bool Resync = false;
+
+ /// A range that one leaf may copy while another leaf, which the region tree
+ /// does not order with it, uses the range. Found when the plan is built; a
+ /// replay in which the copier does copy the range is declined to the host.
+ struct CopyConflictTy {
+ uint32_t Range;
+ uint32_t Copier;
+ uint32_t User;
+ };
+ std::vector<CopyConflictTy> CopyConflicts;
+
+ /// The previous replay was declined for a copy conflict, so another is not
+ /// reported again.
+ bool ConflictDeclined = false;
+
+ /// See declineReported(). Cleared at the start of each acquisition.
+ bool DeclineReported = false;
+};
+
+llvm::ArrayRef<TaskGraphOpTy> TaskGraphNode::ops() const {
+ return {OpPlan, NumOps};
+}
+
+llvm::ArrayRef<TaskGraphKernelParamTy>
+TaskGraphTargetNode::kernelParams() const {
+ return {Params, NumParams};
+}
+
+/// Create a TaskGraphTransferTy describing geometry of a copy operation from
+/// a taskgraph \p Leaf and a particular operation \p Op.
+
+inline TaskGraphTransferTy
+TaskGraphDataPlanTy::transfer(const TaskGraphNode &Leaf,
+ const TaskGraphOpTy &Op) const {
+ using KindTy = TaskGraphOpTy::KindTy;
+ assert(Op.isCopyLike() && "not a copy-like operation");
+ const TaskGraphArgSlotTy &ArgSlot = Leaf.argPlan()[Op.Arg];
+ // Offset non-nullptr Base, or return nullptr.
+ auto maybeOffset = [](void *Base, uint64_t Offset) -> char * {
+ return Base ? static_cast<char *>(Base) + Offset : nullptr;
+ };
+ TaskGraphTransferTy T;
+ T.Enabled = false;
+ auto *Arg = cast<TaskGraphArgBaseTy>(&ArgSlot.get());
+ switch (Op.Kind) {
+ case KindTy::CopyIn:
+ case KindTy::CopyOut: {
+ const auto *M = cast<TaskGraphMappedArgTy>(Arg);
+ char *Device = maybeOffset(M->Range->DevicePtr, M->Offset);
+ char *Host = maybeOffset(M->Range->HostBegin, M->Offset);
+ T.ToDevice = Op.Kind == KindTy::CopyIn;
+ T.HostRange = M->Range;
+ T.Dst = T.ToDevice ? Device : Host;
+ T.Src = T.ToDevice ? Host : Device;
+ T.Size = M->Size;
+ T.Enabled = T.ToDevice ? M->In.Enabled : M->Out.Enabled;
+ break;
+ }
+ case KindTy::Attach: {
+ const auto *A = cast<TaskGraphAttachArgTy>(Arg);
+ T.Dst = maybeOffset(A->PtrRange->DevicePtr, A->Offset);
+ T.Src = attachStagingFor(*A);
+ T.Size = A->Size;
+ T.Enabled = A->Enabled;
+ break;
+ }
+ case KindTy::PrivateInit: {
+ const auto *F = cast<TaskGraphFirstPrivateArgTy>(Arg);
+ T.Dst = PrivateBlock ? privateDevicePtr(*F) : nullptr;
+ T.Src = privateStagingFor(*F);
+ T.Size = F->Size;
+ T.Enabled = true;
+ break;
+ }
+ case KindTy::Launch:
+ break;
+ }
+ return T;
+}
+
+inline uint64_t
+TaskGraphDataPlanTy::paramValue(const TaskGraphTargetNode &Leaf,
+ const TaskGraphKernelParamTy &Param,
+ void *LaunchEnvironment) const {
+ using KindTy = TaskGraphKernelParamTy::KindTy;
+ switch (Param.Kind) {
+ case KindTy::Mapped: {
+ const TaskGraphArgSlotTy &S = Leaf.argPlan()[Param.Arg];
+ const auto *M = cast<TaskGraphMappedArgTy>(&S.get());
+ return reinterpret_cast<uint64_t>(M->Range->DevicePtr) + Param.Offset;
+ }
+ case KindTy::Private: {
+ const TaskGraphArgSlotTy &S = Leaf.argPlan()[Param.Arg];
+ const auto *F = cast<TaskGraphFirstPrivateArgTy>(&S.get());
+ return reinterpret_cast<uint64_t>(privateDevicePtr(*F)) + Param.Offset;
+ }
+ case KindTy::Literal:
+ return reinterpret_cast<uint64_t>(
+ static_cast<const KernelArgsTy *>(Leaf.KernelArgs)
+ ->ArgBasePtrs[Param.Arg]);
+ case KindTy::LaunchEnvironment:
+ return reinterpret_cast<uint64_t>(LaunchEnvironment);
+ }
+ return 0;
+}
+
//===----------------------------------------------------------------------===//
// The graph
//===----------------------------------------------------------------------===//
@@ -292,6 +935,11 @@ class TaskGraphTy {
/// Filled in by libomptarget before the graph is offered to a plugin.
TaskGraphServicesTy Services;
+ /// Where the graph's mapped data lives. Built before the graph is offered to
+ /// a plugin and kept current by libomptarget across replays, so a plugin only
+ /// ever reads it.
+ TaskGraphDataPlanTy Plan;
+
/// The top of the tree: the first element streamed outside any region. A
/// region in general, but a graph holding a single leaf arrives as that leaf.
TaskGraphElement *getRoot() { return Root; }
diff --git a/offload/include/omptarget.h b/offload/include/omptarget.h
index cb98f16c85086e..e0cd2e8a046d5c 100644
--- a/offload/include/omptarget.h
+++ b/offload/include/omptarget.h
@@ -102,6 +102,30 @@ enum tgt_map_type : uint64_t {
namespace omptarget {
namespace maptype {
+/// Copy host->device on region entry (the "to" map modifier).
+inline bool isTo(int64_t MapType) { return MapType & OMP_TGT_MAPTYPE_TO; }
+/// Copy device->host on region exit (the "from" map modifier).
+inline bool isFrom(int64_t MapType) { return MapType & OMP_TGT_MAPTYPE_FROM; }
+/// Transfer regardless of the reference count ("always").
+inline bool isAlways(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_ALWAYS;
+}
+/// Force unmapping / deletion of the device allocation.
+inline bool isDelete(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_DELETE;
+}
+/// The argument is passed to the kernel (its device base address is an arg).
+inline bool isTargetParam(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_TARGET_PARAM;
+}
+/// Pointer-and-pointee map (both the pointer and what it points at are mapped).
+inline bool isPtrAndObj(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_PTR_AND_OBJ;
+}
+/// Return the device base address of the mapped data to the host.
+inline bool isReturnParam(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_RETURN_PARAM;
+}
/// Private (firstprivate aggregate) variable -- not mapped, needs its own
/// per-launch device storage.
inline bool isPrivate(int64_t MapType) {
@@ -111,12 +135,57 @@ inline bool isPrivate(int64_t MapType) {
inline bool isLiteral(int64_t MapType) {
return MapType & OMP_TGT_MAPTYPE_LITERAL;
}
+/// The mapping was added implicitly by the compiler.
+inline bool isImplicit(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_IMPLICIT;
+}
+/// Allocate device memory for the data even under unified shared memory.
+inline bool isClose(int64_t MapType) { return MapType & OMP_TGT_MAPTYPE_CLOSE; }
+/// Runtime error if the data is not already present on the device.
+inline bool isPresent(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_PRESENT;
+}
+/// Attach-pointer map (pointer attachment, processed after all other maps).
+inline bool isAttach(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_ATTACH;
+}
/// Non-contiguous descriptor (used by non-contiguous target update).
inline bool isNonContig(int64_t MapType) {
return MapType & OMP_TGT_MAPTYPE_NON_CONTIG;
}
/// Firstprivate aggregate whose clause carried the "saved" modifier.
inline bool isSaved(int64_t MapType) { return MapType & OMP_TGT_MAPTYPE_SAVED; }
+/// True iff this map is a member of a struct (the struct index is encoded in
+/// the top 16 bits).
+inline bool isMemberOf(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_MEMBER_OF;
+}
+/// The `ompx_hold` extension: the mapping is held for the region's lifetime on
+/// the hold reference count, which `delete` does not reset.
+inline bool isOmpxHold(int64_t MapType) {
+ return MapType & OMP_TGT_MAPTYPE_OMPX_HOLD;
+}
+
+/// Direction of the data movement implied by a map clause, derived once from
+/// the to/from bits, for consumers that want to build transfer nodes.
+enum class TransferDir {
+ None, ///< neither to nor from (alloc-only / pass-by-value / private).
+ H2D, ///< host -> device (to, but not from).
+ D2H, ///< device -> host (from, but not to).
+ Both, ///< to and from (copy in before, copy out after).
+};
+
+inline TransferDir transferDir(int64_t MapType) {
+ bool T = isTo(MapType), F = isFrom(MapType);
+ if (T && F)
+ return TransferDir::Both;
+ if (T)
+ return TransferDir::H2D;
+ if (F)
+ return TransferDir::D2H;
+ return TransferDir::None;
+}
+
} // namespace maptype
} // namespace omptarget
#endif // __cplusplus
diff --git a/offload/libompaccsupport/Mapping.cpp b/offload/libompaccsupport/Mapping.cpp
index 37ec36670d6fd2..b019a751b1fd5b 100644
--- a/offload/libompaccsupport/Mapping.cpp
+++ b/offload/libompaccsupport/Mapping.cpp
@@ -72,6 +72,8 @@ int MappingInfoTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin,
}
// Mapping does not exist, allocate it with refCount=INF
+ warnMappingOverTaskGraphClaim(HstPtrBegin, Size,
+ "omp_target_associate_ptr of");
const HostDataToTargetTy &NewEntry =
*HDTTMap
->emplace(new HostDataToTargetTy(
@@ -83,6 +85,7 @@ int MappingInfoTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin,
/*UseHoldRefCount=*/false, /*Name=*/nullptr,
/*IsRefCountINF=*/true))
.first->HDTT;
+ notifyMappingsChanged();
ODBG(ODT_Mapping) << "Creating new map entry: HstBase="
<< reinterpret_cast<void *>(NewEntry.HstPtrBase)
<< ", HstBegin="
@@ -127,6 +130,7 @@ int MappingInfoTy::disassociatePtr(void *HstPtrBegin) {
if (Event)
Device.destroyEvent(Event);
HDTTMap->erase(It);
+ notifyMappingsChanged();
return Device.notifyDataUnmapped(HstPtrBegin);
}
@@ -296,29 +300,15 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
uintptr_t TgtAllocBegin =
(uintptr_t)Device.allocData(TgtPadding + Size, HstPtrBegin);
uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding;
+ HostDataToTargetTy *Entry =
+ insertMapping(HDTTMap, HstPtrBase, HstPtrBegin, Size, TgtAllocBegin,
+ TgtPtrBegin, HasHoldModifier, HstPtrName);
+ if (!Entry)
+ return TargetPointerResultTy{};
// Release the mapping table lock only after the entry is locked by
// attaching it to TPR.
- LR.TPR.setEntry(HDTTMap
- ->emplace(new HostDataToTargetTy(
- (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
- (uintptr_t)HstPtrBegin + Size, TgtAllocBegin,
- TgtPtrBegin, HasHoldModifier, HstPtrName))
- .first->HDTT);
- INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
- "Creating new map entry with HstPtrBase=" DPxMOD
- ", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD
- ", TgtPtrBegin=" DPxMOD
- ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
- DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin),
- DPxPTR(TgtPtrBegin), Size,
- LR.TPR.getEntry()->dynRefCountToStr().c_str(),
- LR.TPR.getEntry()->holdRefCountToStr().c_str(),
- (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
+ LR.TPR.setEntry(Entry);
LR.TPR.TargetPointer = (void *)TgtPtrBegin;
-
- // Notify the plugin about the new mapping.
- if (Device.notifyDataMapped(HstPtrBegin, Size))
- return TargetPointerResultTy{};
} else {
// This entry is not present and we did not create a new entry for it.
LR.TPR.Flags.IsPresent = false;
@@ -418,6 +408,35 @@ TargetPointerResultTy MappingInfoTy::getTargetPointer(
return std::move(LR.TPR);
}
+HostDataToTargetTy *
+MappingInfoTy::insertMapping(HDTTMapAccessorTy &HDTTMap, void *HstPtrBase,
+ void *HstPtrBegin, int64_t Size,
+ uintptr_t TgtAllocBegin, uintptr_t TgtPtrBegin,
+ bool UseHoldRefCount, map_var_info_t HstPtrName) {
+ warnMappingOverTaskGraphClaim(HstPtrBegin, Size, "mapping");
+ HostDataToTargetTy *Entry =
+ HDTTMap
+ ->emplace(new HostDataToTargetTy(
+ (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
+ (uintptr_t)HstPtrBegin + Size, TgtAllocBegin, TgtPtrBegin,
+ UseHoldRefCount, HstPtrName))
+ .first->HDTT;
+ notifyMappingsChanged();
+ INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
+ "Creating new map entry with HstPtrBase=" DPxMOD ", HstPtrBegin=" DPxMOD
+ ", TgtAllocBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
+ ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
+ DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin),
+ DPxPTR(TgtPtrBegin), Size, Entry->dynRefCountToStr().c_str(),
+ Entry->holdRefCountToStr().c_str(),
+ (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
+
+ // Notify the plugin about the new mapping.
+ if (Device.notifyDataMapped(HstPtrBegin, Size))
+ return nullptr;
+ return Entry;
+}
+
TargetPointerResultTy MappingInfoTy::getTgtPtrBegin(
void *HstPtrBegin, int64_t Size, bool UpdateRefCount, bool UseHoldRefCount,
bool MustContain, bool ForceDelete, bool FromDataEnd) {
@@ -526,6 +545,7 @@ int MappingInfoTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
REPORT() << "Trying to remove a non-existent map entry";
return OFFLOAD_FAIL;
}
+ notifyMappingsChanged();
return OFFLOAD_SUCCESS;
}
@@ -593,3 +613,111 @@ void MappingInfoTy::printCopyInfo(void *TgtPtrBegin, void *HstPtrBegin,
printCopyInfoImpl(Device.DeviceID, H2D, HstPtrBegin, TgtPtrBegin, Size, Entry,
Loc, Name);
}
+
+void MappingInfoTy::splitTaskGraphClaim(uintptr_t Addr) {
+ auto It = TaskGraphClaims.upper_bound(Addr);
+ if (It == TaskGraphClaims.begin())
+ return;
+ --It;
+ if (It->first == Addr || It->second.Hi <= Addr)
+ return;
+ TaskGraphClaimTy Tail = It->second;
+ It->second.Hi = Addr;
+ TaskGraphClaims.emplace(Addr, Tail);
+}
+
+std::map<uintptr_t, MappingInfoTy::TaskGraphClaimTy>::iterator
+MappingInfoTy::firstTaskGraphClaimAfter(uintptr_t Lo) {
+ auto It = TaskGraphClaims.upper_bound(Lo);
+ if (It != TaskGraphClaims.begin() && std::prev(It)->second.Hi > Lo)
+ --It;
+ return It;
+}
+
+void MappingInfoTy::addTaskGraphClaim(const void *Begin, int64_t Size) {
+ const uintptr_t Lo = reinterpret_cast<uintptr_t>(Begin);
+ const uintptr_t Hi = Lo + Size;
+ if (Lo == Hi)
+ return;
+ std::lock_guard<std::mutex> LG(TaskGraphClaimsMtx);
+ splitTaskGraphClaim(Lo);
+ splitTaskGraphClaim(Hi);
+ // Every piece in [Lo, Hi) now lies wholly inside it, so walking the claim
+ // either lands on a piece to count or on a gap to fill.
+ for (uintptr_t At = Lo; At < Hi;) {
+ auto It = TaskGraphClaims.lower_bound(At);
+ if (It != TaskGraphClaims.end() && It->first == At) {
+ ++It->second.Claims;
+ At = It->second.Hi;
+ continue;
+ }
+ const uintptr_t Next =
+ It != TaskGraphClaims.end() && It->first < Hi ? It->first : Hi;
+ TaskGraphClaimTy Piece;
+ Piece.Hi = Next;
+ Piece.Claims = 1;
+ TaskGraphClaims.emplace(At, Piece);
+ At = Next;
+ }
+ NumTaskGraphClaims.store(TaskGraphClaims.size(), std::memory_order_release);
+}
+
+void MappingInfoTy::removeTaskGraphClaim(const void *Begin, int64_t Size) {
+ const uintptr_t Lo = reinterpret_cast<uintptr_t>(Begin);
+ const uintptr_t Hi = Lo + Size;
+ if (Lo == Hi)
+ return;
+ std::lock_guard<std::mutex> LG(TaskGraphClaimsMtx);
+ splitTaskGraphClaim(Lo);
+ splitTaskGraphClaim(Hi);
+ for (auto It = TaskGraphClaims.lower_bound(Lo);
+ It != TaskGraphClaims.end() && It->first < Hi;) {
+ assert(It->second.Claims && "removing a claim that was never added");
+ if (--It->second.Claims == 0)
+ It = TaskGraphClaims.erase(It);
+ else
+ ++It;
+ }
+ NumTaskGraphClaims.store(TaskGraphClaims.size(), std::memory_order_release);
+}
+
+bool MappingInfoTy::warnIfClaimedByTaskGraph(const void *Begin, int64_t Size,
+ const char *What) {
+ if (!NumTaskGraphClaims.load(std::memory_order_acquire))
+ return false;
+ const uintptr_t Lo = reinterpret_cast<uintptr_t>(Begin);
+ const uintptr_t Hi = Lo + std::max<int64_t>(Size, 1);
+ std::lock_guard<std::mutex> LG(TaskGraphClaimsMtx);
+ bool Claimed = false, Warn = false;
+ for (auto It = firstTaskGraphClaimAfter(Lo);
+ It != TaskGraphClaims.end() && It->first < Hi; ++It) {
+ Claimed = true;
+ Warn |= TaskGraphLookupsWarned.insert(It->first).second;
+ }
+ if (Warn)
+ MESSAGE("%s asked about " DPxMOD ", which a taskgraph keeps on the device "
+ "only while it replays, with no mapping outside it; answering as "
+ "if it were not mapped",
+ What, DPxPTR(Begin));
+ return Claimed;
+}
+
+void MappingInfoTy::warnMappingOverTaskGraphClaim(const void *Begin,
+ int64_t Size,
+ const char *What) {
+ if (!NumTaskGraphClaims.load(std::memory_order_acquire))
+ return;
+ const uintptr_t Lo = reinterpret_cast<uintptr_t>(Begin);
+ const uintptr_t Hi = Lo + std::max<int64_t>(Size, 1);
+ std::lock_guard<std::mutex> LG(TaskGraphClaimsMtx);
+ bool Warn = false;
+ for (auto It = firstTaskGraphClaimAfter(Lo);
+ It != TaskGraphClaims.end() && It->first < Hi; ++It) {
+ Warn |= TaskGraphMapsWarned.insert(It->first).second;
+ }
+ if (Warn)
+ MESSAGE("%s " DPxMOD " (%" PRId64 " bytes) overlaps memory a taskgraph "
+ "replay running now keeps on the device for itself; the two "
+ "device copies race",
+ What, DPxPTR(Begin), Size);
+}
diff --git a/offload/libompaccsupport/PluginManager.cpp b/offload/libompaccsupport/PluginManager.cpp
index b49ace3404443d..3b46c5b11bc8ad 100644
--- a/offload/libompaccsupport/PluginManager.cpp
+++ b/offload/libompaccsupport/PluginManager.cpp
@@ -611,6 +611,7 @@ int PluginManager::loadImagesOntoDevice(DeviceTy &Device) {
(uintptr_t)CurrDeviceEntryAddr /*TgtPtrBegin*/,
false /*UseHoldRefCount*/, CurrHostEntry->SymbolName,
true /*IsRefCountINF*/));
+ Device.getMappingInfo().notifyMappingsChanged();
// Notify about the new mapping.
if (Device.notifyDataMapped(CurrHostEntry->Address,
diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index a05db18a341d7a..ad4450d6a433e0 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -11,6 +11,7 @@ add_library(omptarget SHARED
LegacyAPI.cpp
OpenMP/API.cpp
+ OpenMP/TaskGraphPlan.cpp
OpenMP/InteropAPI.cpp
OpenMP/OMPT/Callback.cpp
diff --git a/offload/libomptarget/OpenMP/API.cpp b/offload/libomptarget/OpenMP/API.cpp
index 4a8e21e5303270..dab9e375fc7d44 100644
--- a/offload/libomptarget/OpenMP/API.cpp
+++ b/offload/libomptarget/OpenMP/API.cpp
@@ -246,6 +246,8 @@ EXTERN int omp_target_is_present(const void *Ptr, int DeviceNum) {
/*UpdateRefCount=*/false,
/*UseHoldRefCount=*/false);
int Rc = TPR.isPresent();
+ if (!Rc)
+ DeviceOrErr->getMappingInfo().warnIfClaimedByTaskGraph(Ptr, 1, __func__);
ODBG(ODT_Interface) << "Call to " << __func__ << " returns " << Rc;
return Rc;
}
@@ -281,6 +283,17 @@ EXTERN int omp_target_is_accessible(const void *Ptr, size_t Size,
if (!DeviceOrErr)
FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
+ // Memory a taskgraph replay keeps on the device for itself is not accessible
+ // to the program, unless the program has mapped it as well.
+ MappingInfoTy &MappingInfo = DeviceOrErr->getMappingInfo();
+ if (MappingInfo.hasTaskGraphClaims() &&
+ !MappingInfo
+ .getTgtPtrBegin(const_cast<void *>(Ptr), Size,
+ /*UpdateRefCount=*/false, /*UseHoldRefCount=*/false)
+ .isPresent() &&
+ MappingInfo.warnIfClaimedByTaskGraph(Ptr, Size, __func__))
+ return false;
+
return DeviceOrErr->isAccessiblePtr(Ptr, Size);
}
@@ -750,6 +763,8 @@ EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) {
if (!TPR.isPresent()) {
ODBG(ODT_Interface) << "Ptr " << Ptr
<< "is not present on device %d, returning nullptr.";
+ TPR.reset();
+ DeviceOrErr->getMappingInfo().warnIfClaimedByTaskGraph(Ptr, 1, __func__);
return nullptr;
}
diff --git a/offload/libomptarget/OpenMP/TaskGraphPlan.cpp b/offload/libomptarget/OpenMP/TaskGraphPlan.cpp
new file mode 100644
index 00000000000000..f387edc4502143
--- /dev/null
+++ b/offload/libomptarget/OpenMP/TaskGraphPlan.cpp
@@ -0,0 +1,1751 @@
+//===-- OpenMP/TaskGraphPlan.cpp - A taskgraph's mapped data --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "OpenMP/TaskGraphPlan.h"
+
+#include "OpenMP/Mapping.h"
+#include "PluginManager.h"
+#include "Shared/APITypes.h"
+#include "Shared/Debug.h"
+#include "Shared/EnvironmentVar.h"
+#include "device.h"
+#include "omptarget.h"
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/MathExtras.h"
+
+#include <algorithm>
+#include <cstring>
+#include <map>
+#include <vector>
+
+using namespace llvm;
+using namespace llvm::omp::target;
+
+namespace mt = omptarget::maptype;
+
+/// Report every replay that finds the host captures relocated or the mapping
+/// state changed since the previous one. Off by default; tests use it to check
+/// that a change they set up was noticed.
+static bool reportDrift() {
+ static BoolEnvar ReportDrift("LIBOMPTARGET_TASKGRAPH_REPORT_DRIFT", false);
+ return ReportDrift;
+}
+
+namespace llvm {
+namespace omp {
+namespace target {
+
+/// Builds a graph's data plan and keeps it current across replays: everything
+/// that reads or writes the plan's own state, which a backend cannot see.
+class TaskGraphPlanner {
+public:
+ static bool buildTaskGraphPlan(DeviceTy &Device, TaskGraphTy &Graph);
+ static bool
+ acquireTaskGraphMappings(DeviceTy &Device, TaskGraphTy &Graph,
+ SmallVectorImpl<TaskGraphArgUpdateTy> &Updates,
+ bool &Resync);
+ static void commitTaskGraphUpdates(TaskGraphTy &Graph);
+ static void releaseTaskGraphMappings(DeviceTy &Device, TaskGraphTy &Graph,
+ bool Replayed = true);
+ static void beginTaskGraphReplay(TaskGraphTy &Graph);
+ static void releaseTaskGraphPlan(DeviceTy &Device, TaskGraphTy &Graph);
+
+private:
+ /// What a map does to its region's reference count, which is what decides
+ /// whether the map performs its implicit copies.
+ enum class ClaimKindTy : uint8_t {
+ /// A `target` region's map: takes a reference for the duration of the
+ /// region and gives it back, so one map both creates the mapping and
+ /// removes it.
+ Balanced,
+ /// `target enter data`: takes a reference and keeps it.
+ Enter,
+ /// `target exit data`: gives one back.
+ Exit,
+ /// `target update`, or a map naming a region the graph does not map itself.
+ /// It copies explicitly or not at all, and either way the count is not its
+ /// business.
+ Lookup,
+ };
+
+ /// One map naming one region, in encounter order, so that the reference count
+ /// over each range can be walked once the maps have been gathered.
+ struct ClaimTy {
+ ClaimKindTy Kind = ClaimKindTy::Lookup;
+ /// The argument the map came from.
+ TaskGraphArgSlotTy *Arg = nullptr;
+ char *Begin = nullptr;
+ uint64_t Size = 0;
+ int64_t ArgType = 0;
+ /// For a struct member's map, the map of its enclosing struct. The member
+ /// shares the struct's mapping and leaves the reference count alone, so it
+ /// copies on whichever change the struct's map owns.
+ const TaskGraphArgSlotTy *Parent = nullptr;
+ };
+
+ /// The ranges being gathered while a plan is built, keyed by host begin.
+ using RangeMapTy = std::map<char *, TaskGraphRangeTy>;
+
+ static MutableArrayRef<TaskGraphArgSlotTy> argsOf(TaskGraphNode *Node) {
+ return {Node->ArgPlan, Node->NumArgPlans};
+ }
+
+ static bool hostResidentIfUnmapped(const DeviceTy &Device,
+ const TaskGraphRangeTy &Range);
+ static void reportUnresolvable(DeviceTy &Device,
+ const TaskGraphRangeTy &Range);
+ static TaskGraphRangeTy *
+ containingRange(MutableArrayRef<TaskGraphRangeTy> Ranges, char *Begin,
+ uint64_t Size);
+ static void gatherClaim(RangeMapTy &Ranges, const ClaimTy &C);
+ static bool rekeyRanges(TaskGraphTy &Graph, SmallVectorImpl<bool> &HostMoved);
+ static void *allocateRangeStorage(DeviceTy &Device,
+ const TaskGraphRangeTy &Range);
+ static bool publishRange(DeviceTy &Device,
+ MappingInfoTy::HDTTMapAccessorTy &HDTTMap,
+ const TaskGraphRangeTy &Range, void *Storage);
+ static bool acquireRange(DeviceTy &Device, TaskGraphRangeTy &Range,
+ void *&Address, bool &Inserted, bool &HostResident);
+ static void releaseReference(DeviceTy &Device, TaskGraphRangeTy &Range,
+ bool UseHoldRefCount, bool ForceDelete,
+ bool MayRemove);
+ static void releaseRange(DeviceTy &Device, TaskGraphRangeTy &Range);
+ static bool pointeeDeviceBase(const TaskGraphRangeTy &Pointee,
+ int64_t PointeeOffset, char *HstPteeBase,
+ char *HstPteeBegin, char *&TgtPteeBase);
+ static bool stageTaskGraphAttachments(DeviceTy &Device, TaskGraphTy &Graph,
+ MutableArrayRef<uint8_t> Changes);
+ static void restoreTaskGraphAttachments(TaskGraphTy &Graph);
+ static bool copiesThisReplay(TaskGraphNode *Node, uint32_t Range);
+ static bool buildOpsAndParams(TaskGraphTy &Graph);
+};
+
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+/// Why a map cannot be lowered to a device buffer plus transfers, or null if it
+/// can. Struct members and pointer-and-pointee entries need nothing special:
+/// once relocated they are a host address and an extent like any other, and
+/// gathering puts a member in the range of its enclosing struct. Attachments
+/// and privatised arguments are classified before this is asked.
+static const char *whyUnlowerable(int64_t ArgType, void *Mapper) {
+ if (Mapper)
+ return "uses a user-defined mapper";
+ if (mt::isNonContig(ArgType))
+ return "is a non-contiguous transfer";
+ return nullptr;
+}
+
+/// Whether an argument is the fix-up for a lambda's by-reference capture, which
+/// writes the captured variable's device address into the device copy of the
+/// lambda. It looks like a literal, but the plan has no operation that writes
+/// into an argument's device copy, so a graph with one cannot be lowered.
+static bool isLambdaCaptureFixup(int64_t ArgType) {
+ constexpr int64_t Fixup = OMP_TGT_MAPTYPE_PTR_AND_OBJ |
+ OMP_TGT_MAPTYPE_LITERAL | OMP_TGT_MAPTYPE_IMPLICIT;
+ return !mt::isTargetParam(ArgType) && (ArgType & Fixup) == Fixup;
+}
+
+/// Whether an argument is a pointer attachment: a device pointer to be pointed
+/// at a device pointee once both are mapped. `PRIVATE | ATTACH` is not one; it
+/// initialises a privatised pointer, so it is classified as a private.
+static bool isAttachment(int64_t ArgType) {
+ return mt::isAttach(ArgType) && !mt::isPrivate(ArgType);
+}
+
+/// Whether an argument is privatised: storage for the construct alone, which
+/// the graph provides rather than maps.
+static bool isPrivatised(int64_t ArgType) { return mt::isPrivate(ArgType); }
+
+/// The index of the argument mapping the struct that argument \p ArgType is a
+/// member of, or -1 if it is not a member whose copies follow that struct's
+/// reference count. A pointer-and-pointee member counts on its pointee, as in
+/// targetDataBegin.
+static int64_t structParentIndex(int64_t ArgType) {
+ if (!mt::isMemberOf(ArgType) || mt::isPtrAndObj(ArgType))
+ return -1;
+ return ((ArgType & OMP_TGT_MAPTYPE_MEMBER_OF) >> 48) - 1;
+}
+
+/// Whether child \p To of an irreducible region can be reached from child
+/// \p From along the region's edges.
+static bool reaches(const TaskGraphIrreducibleRegion &Region, uint32_t From,
+ uint32_t To) {
+ SmallVector<bool, 16> Seen(Region.NumChildren, false);
+ SmallVector<uint32_t, 16> Work = {From};
+ Seen[From] = true;
+ while (!Work.empty()) {
+ const uint32_t Child = Work.pop_back_val();
+ for (const TaskGraphEdge &E : Region.edges()) {
+ if (E.Src != static_cast<int32_t>(Child) || E.Dst < 0 ||
+ static_cast<uint32_t>(E.Dst) >= Region.NumChildren)
+ continue;
+ if (static_cast<uint32_t>(E.Dst) == To)
+ return true;
+ if (!Seen[E.Dst]) {
+ Seen[E.Dst] = true;
+ Work.push_back(E.Dst);
+ }
+ }
+ }
+ return false;
+}
+
+/// Whether the region tree orders leaves \p A and \p B one way or the other, so
+/// that they never run at the same time. It does if their lowest common region
+/// is sequential, or is irreducible and has a path between the children leading
+/// to them. It also does for an exclusive region: its children are lowered in
+/// child order, the order the plan's reference-count walk assumes, and the HIP
+/// backend declines such graphs anyway. A parallel region's children are
+/// unordered.
+static bool areOrdered(const TaskGraphElement &A, const TaskGraphElement &B) {
+ auto depth = [](const TaskGraphElement *E) {
+ uint32_t Depth = 0;
+ for (; E->Parent; E = E->Parent)
+ ++Depth;
+ return Depth;
+ };
+ // Lift the deeper one to the other's depth, then both until their parents
+ // meet: the parent is then the lowest common region, and each is its child
+ // leading to the leaf.
+ const TaskGraphElement *X = &A, *Y = &B;
+ uint32_t DepthX = depth(X), DepthY = depth(Y);
+ for (; DepthX > DepthY; --DepthX)
+ X = X->Parent;
+ for (; DepthY > DepthX; --DepthY)
+ Y = Y->Parent;
+ while (X->Parent != Y->Parent) {
+ X = X->Parent;
+ Y = Y->Parent;
+ }
+ assert(X != Y && X->Parent && "two distinct leaves of one graph");
+
+ switch (X->Parent->getKind()) {
+ case TGKind::SeqRegion:
+ case TGKind::ExclRegion:
+ return true;
+ case TGKind::IrreducibleRegion: {
+ const auto &Region = *cast<TaskGraphIrreducibleRegion>(X->Parent);
+ return reaches(Region, X->ChildIndex, Y->ChildIndex) ||
+ reaches(Region, Y->ChildIndex, X->ChildIndex);
+ }
+ default:
+ return false;
+ }
+}
+
+/// Alignment for one entry's slot in the private block: the alignment the host
+/// object happens to have, as PrivateArgumentManager infers it, capped at 16
+/// and floored at a pointer so that a pointer initialisation is one aligned
+/// word.
+static uint64_t privateAlignment(const void *HstPtr) {
+ constexpr uint64_t MaxAlign = 16;
+ const uint64_t Addr = reinterpret_cast<uintptr_t>(HstPtr);
+ const uint64_t Natural = Addr ? (Addr & ~(Addr - 1)) : MaxAlign;
+ return std::clamp<uint64_t>(Natural, sizeof(void *), MaxAlign);
+}
+
+/// Whether \p Range uses the host memory directly when nothing has mapped it:
+/// under `unified_shared_memory` unless a map asked for `close`, and always
+/// under automatic zero-copy. The same rule getTargetPointer applies.
+bool TaskGraphPlanner::hostResidentIfUnmapped(const DeviceTy &Device,
+ const TaskGraphRangeTy &Range) {
+ return ((Device.PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY) &&
+ !Range.Close) ||
+ (Device.PM->getRequirements() & OMPX_REQ_AUTO_ZERO_COPY);
+}
+
+/// A range's end as a void *, so that a report stream prints it as an address
+/// rather than as a string.
+static void *endOf(const TaskGraphRangeTy &Range) { return Range.end(); }
+
+/// Say why a range could not be resolved. Looks the range up again, which
+/// only the failing path pays for.
+void TaskGraphPlanner::reportUnresolvable(DeviceTy &Device,
+ const TaskGraphRangeTy &Range) {
+ MappingInfoTy &MappingInfo = Device.getMappingInfo();
+ auto HDTTMap = MappingInfo.HostDataToTargetMap.getExclusiveAccessor();
+ LookupResult LR = MappingInfo.lookupMapping(HDTTMap, Range.HostBegin,
+ static_cast<int64_t>(Range.Size));
+ const bool Contained = LR.Flags.IsContained;
+ const bool Partial = LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter;
+ LR.TPR.reset();
+ HDTTMap.destroy();
+
+ if (Partial) {
+ // Outside a graph an explicit map of this shape is refused too. An
+ // implicit one may borrow the existing entry, but a range gathered from
+ // several maps has no single map's implicitness to go on.
+ REPORT() << "taskgraph maps [" << Range.HostBegin << ", " << endOf(Range)
+ << ") which partially overlaps a region mapped outside the graph, "
+ "leaving it with neither a device address to share nor one it "
+ "can map without aliasing";
+ return;
+ }
+ if (Range.RequiresPresent && !Contained) {
+ REPORT() << "taskgraph requires [" << Range.HostBegin << ", "
+ << endOf(Range) << ") to be present, and it is not";
+ return;
+ }
+ REPORT() << "taskgraph could not map " << Range.Size << " bytes for ["
+ << Range.HostBegin << ", " << endOf(Range) << ")";
+}
+
+/// The direction a data leaf transfers an argument in. A `target data` region
+/// records its enter and exit actions with one shared map array, so a `tofrom`
+/// map asks for both directions on both; the enter action only copies in and
+/// the exit action only copies out. A map naming neither direction, such as
+/// `alloc` or `release`, transfers nothing.
+static mt::TransferDir directionFor(const TaskGraphTargetDataNode *DataNode,
+ int64_t ArgType) {
+ mt::TransferDir Dir = mt::transferDir(ArgType);
+ if (isa<TaskGraphTargetEnterDataNode>(DataNode))
+ return mt::isTo(ArgType) ? mt::TransferDir::H2D : mt::TransferDir::None;
+ if (isa<TaskGraphTargetExitDataNode>(DataNode))
+ return mt::isFrom(ArgType) ? mt::TransferDir::D2H : mt::TransferDir::None;
+ return Dir;
+}
+
+/// Where a leaf's argument \p Arg currently begins, after relocation.
+static char *argBegin(TaskGraphNode *Node, uint32_t Arg) {
+ if (auto *TargetNode = dyn_cast<TaskGraphTargetNode>(Node))
+ return static_cast<char *>(
+ static_cast<KernelArgsTy *>(TargetNode->KernelArgs)->ArgPtrs[Arg]);
+ return static_cast<char *>(cast<TaskGraphTargetDataNode>(Node)->Args[Arg]);
+}
+
+/// Where a leaf's argument \p Arg has its base, which for an attach entry is
+/// the pointer object.
+static char *argBase(TaskGraphNode *Node, uint32_t Arg) {
+ if (auto *TargetNode = dyn_cast<TaskGraphTargetNode>(Node))
+ return static_cast<char *>(
+ static_cast<KernelArgsTy *>(TargetNode->KernelArgs)->ArgBasePtrs[Arg]);
+ return static_cast<char *>(
+ cast<TaskGraphTargetDataNode>(Node)->ArgsBase[Arg]);
+}
+
+/// How many bytes a leaf's argument \p Arg currently covers.
+static uint64_t argSize(TaskGraphNode *Node, uint32_t Arg) {
+ if (auto *TargetNode = dyn_cast<TaskGraphTargetNode>(Node))
+ return static_cast<KernelArgsTy *>(TargetNode->KernelArgs)->ArgSizes[Arg];
+ return cast<TaskGraphTargetDataNode>(Node)->ArgSizes[Arg];
+}
+
+/// Whether two host extents have to share a range. Overlap is strict, so
+/// extents that merely abut are kept apart. A zero extent `[p, p)` shares with
+/// an extent that contains `p` or begins at it, so that a zero-length array
+/// section falls into the range mapping its base.
+static bool shareRange(const char *A, uint64_t ASize, const char *B,
+ uint64_t BSize) {
+ if (!ASize && !BSize)
+ return A == B;
+ if (!ASize)
+ return B <= A && A < B + BSize;
+ if (!BSize)
+ return A <= B && B < A + ASize;
+ return A < B + BSize && B < A + ASize;
+}
+
+/// The range holding all of \p Size bytes at \p Begin, or null if none does.
+/// A zero extent is held by a range containing it or beginning at it. The
+/// ranges have to be in host-address order, as they are when the plan is
+/// built.
+TaskGraphRangeTy *
+TaskGraphPlanner::containingRange(MutableArrayRef<TaskGraphRangeTy> Ranges,
+ char *Begin, uint64_t Size) {
+ auto *It =
+ llvm::upper_bound(Ranges, Begin, [](char *B, const TaskGraphRangeTy &R) {
+ return B < R.begin();
+ });
+ if (It == Ranges.begin())
+ return nullptr;
+ TaskGraphRangeTy &Range = *std::prev(It);
+ if (!Size)
+ return Begin < Range.end() || Begin == Range.begin() ? &Range : nullptr;
+ return Begin + Size <= Range.end() ? &Range : nullptr;
+}
+
+/// Gather one map into the ranges, merging it with every range it shares host
+/// memory with. This mirrors the mapping table, where a map contained in an
+/// existing entry reuses that entry's device memory, so a `target` region
+/// mapping part of an enclosing `target data` region joins its range. Ranges
+/// that merely abut stay apart, as separate mappings would.
+void TaskGraphPlanner::gatherClaim(RangeMapTy &Ranges, const ClaimTy &C) {
+ TaskGraphRangeTy Merged;
+ Merged.RefDelta = C.Parent ? 0
+ : C.Kind == ClaimKindTy::Enter ? 1
+ : C.Kind == ClaimKindTy::Exit ? -1
+ : 0;
+ Merged.RequiresPresent = mt::isPresent(C.ArgType);
+ // A `present` map never creates the mapping. Another map in the graph may,
+ // in which case the range is present by the time the `present` map runs.
+ Merged.MayCreate =
+ (C.Kind == ClaimKindTy::Balanced || C.Kind == ClaimKindTy::Enter) &&
+ !Merged.RequiresPresent;
+ Merged.Deleted = mt::isDelete(C.ArgType);
+ Merged.Holds = mt::isOmpxHold(C.ArgType);
+ Merged.Close = mt::isClose(C.ArgType);
+
+ char *Lo = C.Begin;
+ char *Hi = C.Begin + C.Size;
+ // The ranges are disjoint and ordered, so only the one before the insertion
+ // point can reach back over the claim's begin, and the ones the claim shares
+ // with follow it. The merged extent may bridge ranges that were apart.
+ auto It = Ranges.upper_bound(Lo);
+ if (It != Ranges.begin() &&
+ shareRange(std::prev(It)->first, std::prev(It)->second.Size, Lo, Hi - Lo))
+ --It;
+ while (It != Ranges.end() &&
+ shareRange(It->first, It->second.Size, Lo, Hi - Lo)) {
+ const TaskGraphRangeTy &R = It->second;
+ Lo = std::min(Lo, R.begin());
+ Hi = std::max(Hi, R.end());
+ Merged.RefDelta += R.RefDelta;
+ Merged.MayCreate |= R.MayCreate;
+ Merged.RequiresPresent |= R.RequiresPresent;
+ Merged.Deleted |= R.Deleted;
+ Merged.Holds |= R.Holds;
+ Merged.Close |= R.Close;
+ It = Ranges.erase(It);
+ }
+ Merged.HostBegin = Lo;
+ Merged.Size = Hi - Lo;
+ Ranges.emplace(Lo, Merged);
+}
+
+/// Move the ranges to wherever relocation has just put the captures they were
+/// resolved from, and set \p HostMoved, indexed by range Id, for the ones that
+/// moved.
+///
+/// The grouping is fixed, so a range may only move as a whole: every argument
+/// in it must have moved by the same distance, and the moved ranges must not
+/// overlap one another. A relocation that breaks this, for instance by moving
+/// only one of two leaves mapping the same object, is refused and leaves the
+/// plan unchanged.
+bool TaskGraphPlanner::rekeyRanges(TaskGraphTy &Graph,
+ SmallVectorImpl<bool> &HostMoved) {
+ TaskGraphDataPlanTy &Plan = Graph.Plan;
+ const uint64_t Stamp = ++Plan.Relocations;
+
+ // Extents are fixed as well, since a lowered transfer cannot grow. An
+ // argument whose size changed, such as a reallocated Fortran allocatable, is
+ // refused.
+ auto resized = [&](const TaskGraphRangeTy &R, uint64_t Was, uint64_t Now) {
+ if (Was == Now)
+ return false;
+ REPORT() << "taskgraph replay found the region [" << R.HostBegin << ", "
+ << endOf(R) << ") covering " << Now
+ << " bytes where the recording had " << Was
+ << "; a graph's mapped extents are fixed when it is recorded";
+ return true;
+ };
+
+ // The first argument to reach a range in this relocation records the
+ // distance and later ones are checked against it. A zero-length entry whose
+ // address did not move is exempt: its address can come from a slot the
+ // relocation helper leaves alone, such as a static pointer into a stack
+ // array, and it simply keeps its offset in the range.
+ auto note = [&](TaskGraphRangeTy &R, char *Now, uint64_t Offset,
+ uint64_t Size) {
+ const int64_t D = Now - (R.begin() + Offset);
+ if (!Size && !D)
+ return true;
+ if (R.MovedInRelocation != Stamp) {
+ R.MovedInRelocation = Stamp;
+ R.MovedBy = D;
+ return true;
+ }
+ if (R.MovedBy == D)
+ return true;
+ REPORT() << "taskgraph relocation moved part of the region [" << R.HostBegin
+ << ", " << endOf(R) << ") without moving the rest of it";
+ return false;
+ };
+
+ // Check every entry naming a range as its own before moving anything. A
+ // pointee is only looked up, and is re-read every replay, so it is not
+ // checked.
+ for (TaskGraphNode *Node : Graph.leaves()) {
+ for (auto [I, S] : llvm::enumerate(argsOf(Node))) {
+ const uint32_t Arg = static_cast<uint32_t>(I);
+ const TaskGraphArgBaseTy *A = &S.get();
+ char *Now;
+ TaskGraphRangeTy *Range;
+ uint64_t Offset, Size;
+
+ switch (A->getKind()) {
+ case TaskGraphArgBaseTy::KindTy::Mapped: {
+ Now = argBegin(Node, Arg);
+ const auto *M = cast<TaskGraphMappedArgTy>(A);
+ Range = M->Range;
+ Size = M->Size;
+ Offset = M->Offset;
+ break;
+ }
+ case TaskGraphArgBaseTy::KindTy::Attach: {
+ Now = argBase(Node, Arg);
+ const auto *T = cast<TaskGraphAttachArgTy>(A);
+ Range = T->PtrRange;
+ Size = T->Size;
+ Offset = T->Offset;
+ break;
+ }
+ default:
+ continue;
+ }
+
+ if (resized(*Range, Size, argSize(Node, Arg)) ||
+ !note(*Range, Now, Offset, Size))
+ return false;
+ }
+ }
+
+ auto moves = [&](const TaskGraphRangeTy &R) {
+ return R.MovedInRelocation == Stamp && R.MovedBy;
+ };
+ auto newBegin = [&](const TaskGraphRangeTy &R) {
+ return moves(R) ? R.begin() + R.MovedBy : R.begin();
+ };
+
+ // The moved ranges must still be disjoint from one another and from the rest.
+ // Check before moving anything, so a refusal leaves the plan as it was.
+ SmallVector<const TaskGraphRangeTy *, 16> Sorted;
+ bool AnyMoves = false;
+ for (const TaskGraphRangeTy &Range : Plan.Ranges) {
+ Sorted.push_back(&Range);
+ AnyMoves |= moves(Range);
+ }
+ if (!AnyMoves)
+ return true;
+ llvm::sort(Sorted, [&](const TaskGraphRangeTy *A, const TaskGraphRangeTy *B) {
+ return newBegin(*A) < newBegin(*B);
+ });
+ for (size_t I = 1; I < Sorted.size(); ++I) {
+ const TaskGraphRangeTy &Prev = *Sorted[I - 1];
+ const TaskGraphRangeTy &Next = *Sorted[I];
+ // A range of no extent shares its begin with nothing.
+ if (newBegin(Next) != newBegin(Prev) &&
+ newBegin(Next) >= newBegin(Prev) + Prev.Size)
+ continue;
+ const TaskGraphRangeTy &Clash = moves(Next) ? Next : Prev;
+ const TaskGraphRangeTy &Over = moves(Next) ? Prev : Next;
+ REPORT() << "taskgraph relocation put [" << Clash.HostBegin << ", "
+ << endOf(Clash) << ") over [" << Over.HostBegin << ", "
+ << endOf(Over)
+ << "), which were separate regions when the graph was recorded";
+ return false;
+ }
+
+ for (TaskGraphRangeTy &Range : Plan.Ranges)
+ if (moves(Range)) {
+ Range.HostBegin = Range.begin() + Range.MovedBy;
+ HostMoved[Range.Id] = true;
+ }
+ return true;
+}
+
+/// Device storage for a range the graph maps and finds absent.
+void *TaskGraphPlanner::allocateRangeStorage(DeviceTy &Device,
+ const TaskGraphRangeTy &Range) {
+ return Device.allocData(static_cast<int64_t>(Range.Size), Range.HostBegin);
+}
+
+/// Publish \p Storage as the mapping of \p Range, found absent under
+/// \p HDTTMap, holding the graph's hold reference. A range the graph leaves
+/// mapped also gets the reference an unbalanced `target enter data` leaves: on
+/// the hold count under `ompx_hold`, and on the dynamic count otherwise.
+bool TaskGraphPlanner::publishRange(DeviceTy &Device,
+ MappingInfoTy::HDTTMapAccessorTy &HDTTMap,
+ const TaskGraphRangeTy &Range,
+ void *Storage) {
+ const uintptr_t Tgt = reinterpret_cast<uintptr_t>(Storage);
+ HostDataToTargetTy *Entry = Device.getMappingInfo().insertMapping(
+ HDTTMap, /*HstPtrBase=*/Range.HostBegin, Range.HostBegin,
+ static_cast<int64_t>(Range.Size), /*TgtAllocBegin=*/Tgt,
+ /*TgtPtrBegin=*/Tgt, /*UseHoldRefCount=*/true, /*HstPtrName=*/nullptr);
+ if (!Entry)
+ return false;
+ if (Range.persists())
+ Entry->incRefCount(/*UseHoldRefCount=*/Range.Holds);
+ return true;
+}
+
+/// Resolve one range against the mapping table for a replay about to start,
+/// mapping it if the graph brings it into the device data environment, and
+/// take a hold reference so the mapping cannot be removed while the replay
+/// runs. A mapping the graph creates starts with its hold count at one, which
+/// is that reference.
+///
+/// Sets \p Address to the range's device address for this replay, or null if
+/// it is absent and the graph does not map it; \p Inserted if the graph created
+/// the mapping or the storage; and \p HostResident if the device uses the host
+/// memory directly.
+bool TaskGraphPlanner::acquireRange(DeviceTy &Device, TaskGraphRangeTy &Range,
+ void *&Address, bool &Inserted,
+ bool &HostResident) {
+ MappingInfoTy &MappingInfo = Device.getMappingInfo();
+ Address = nullptr;
+ Inserted = false;
+ HostResident = false;
+ Range.DropsLast = false;
+
+ const int64_t Size = static_cast<int64_t>(Range.Size);
+
+ // The exclusive accessor is held until the range is borrowed or published, so
+ // no other thread can map the range in between. A range of no extent only
+ // ever borrows; lookupMapping finds an entry for it by the entry's extended
+ // range, so a section starting before the mapped bytes is still found.
+ auto HDTTMap = MappingInfo.HostDataToTargetMap.getExclusiveAccessor();
+ LookupResult LR = MappingInfo.lookupMapping(HDTTMap, Range.HostBegin, Size);
+
+ // A self-contained range gets storage of its own unless the program has
+ // since mapped it. A mapping containing the range is borrowed below, as for
+ // any other range; a partial overlap declines the replay.
+ if (Range.SelfContained && !LR.Flags.IsContained) {
+ const bool Overlaps = LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter;
+ LR.TPR.reset();
+ HDTTMap.destroy();
+ if (Overlaps) {
+ MESSAGE("taskgraph keeps [" DPxMOD ", " DPxMOD
+ ") on the device for itself, and it is now partly mapped "
+ "outside the graph",
+ DPxPTR(Range.HostBegin), DPxPTR(Range.end()));
+ return false;
+ }
+ void *Storage = allocateRangeStorage(Device, Range);
+ if (!Storage) {
+ REPORT() << "taskgraph could not allocate " << Range.Size
+ << " bytes of device memory for [" << Range.HostBegin << ", "
+ << endOf(Range) << ")";
+ return false;
+ }
+ Address = Storage;
+ Inserted = true;
+ Range.StorageHeld = true;
+ // Claimed only for as long as the storage exists: the host address need
+ // not be the program's once the replay ends.
+ MappingInfo.addTaskGraphClaim(Range.HostBegin, Size);
+ return true;
+ }
+
+ if (LR.Flags.IsContained) {
+ HostDataToTargetTy *Entry = LR.TPR.getEntry();
+ // The graph drops one reference to a mapping it found, which removes it
+ // only if it is the last -- and one with a hold reference is not removed
+ // by a dynamic one at all.
+ Range.DropsLast = Range.releases() && Entry->getHoldRefCount() == 0 &&
+ Entry->getDynRefCount() == 1;
+ Entry->incRefCount(/*UseHoldRefCount=*/true);
+ Address = reinterpret_cast<void *>(
+ Entry->TgtPtrBegin +
+ (reinterpret_cast<uintptr_t>(Range.HostBegin) - Entry->HstPtrBegin));
+ Range.Held = true;
+ HDTTMap.destroy();
+ // Whoever mapped the range may still be copying its contents in, and the
+ // graph's first reader must not overtake that.
+ if (void *Event = Entry->getEvent()) {
+ AsyncInfoTy AsyncInfo(Device);
+ if (Device.waitEvent(Event, AsyncInfo) != OFFLOAD_SUCCESS ||
+ AsyncInfo.synchronize() != OFFLOAD_SUCCESS)
+ return false;
+ }
+ return true;
+ }
+
+ // A partial overlap can be neither borrowed nor mapped.
+ if (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) {
+ LR.TPR.reset();
+ HDTTMap.destroy();
+ reportUnresolvable(Device, Range);
+ return false;
+ }
+ LR.TPR.reset();
+
+ if (hostResidentIfUnmapped(Device, Range)) {
+ HostResident = true;
+ Address = Range.HostBegin;
+ return true;
+ }
+
+ // `present` is only enforced on a range the graph does not map; one it does
+ // map is brought in here, before any of the graph runs.
+ if (!Range.Size || !Range.MayCreate) {
+ if (!Range.RequiresPresent)
+ return true; // absent, and not ours to map: its updates are no-ops.
+ HDTTMap.destroy();
+ reportUnresolvable(Device, Range);
+ return false;
+ }
+
+ void *Storage = allocateRangeStorage(Device, Range);
+ if (!Storage) {
+ REPORT() << "taskgraph could not map " << Range.Size << " bytes for ["
+ << Range.HostBegin << ", " << endOf(Range) << ")";
+ return false;
+ }
+ if (!publishRange(Device, HDTTMap, Range, Storage)) {
+ HDTTMap.destroy();
+ REPORT() << "taskgraph could not publish its mapping of ["
+ << Range.HostBegin << ", " << endOf(Range) << ")";
+ return false;
+ }
+ Inserted = true;
+ Address = Storage;
+ Range.Held = true;
+ return true;
+}
+
+/// Drop one reference to the mapping covering \p Range and, if that was the
+/// last and \p MayRemove is set, remove it. Follows the protocol of
+/// postProcessingTargetDataEnd, so that only the last of several concurrent
+/// releasers frees the entry.
+void TaskGraphPlanner::releaseReference(DeviceTy &Device,
+ TaskGraphRangeTy &Range,
+ bool UseHoldRefCount, bool ForceDelete,
+ bool MayRemove) {
+ MappingInfoTy &MappingInfo = Device.getMappingInfo();
+ const int64_t Size = static_cast<int64_t>(Range.Size);
+
+ TargetPointerResultTy TPR = MappingInfo.getTgtPtrBegin(
+ Range.HostBegin, Size, /*UpdateRefCount=*/true, UseHoldRefCount,
+ /*MustContain=*/true, ForceDelete, /*FromDataEnd=*/true);
+ if (!TPR.isPresent() || !TPR.getEntry())
+ return;
+
+ bool DelEntry = MayRemove && TPR.Flags.IsLast && !TPR.isHostPointer();
+ HostDataToTargetTy *Entry = TPR.getEntry();
+
+ Entry->unlock();
+ MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+ MappingInfo.HostDataToTargetMap.getExclusiveAccessor();
+ Entry->lock();
+ const bool NotLastUser = Entry->decDataEndThreadCount() != 0;
+ if (DelEntry && (Entry->getTotalRefCount() != 0 || NotLastUser))
+ DelEntry = false;
+ TPR.setEntry(nullptr);
+ if (!DelEntry)
+ return;
+
+ int Ret = MappingInfo.eraseMapEntry(HDTTMap, Entry, Size);
+ HDTTMap.destroy();
+ Ret |= MappingInfo.deallocTgtPtrAndEntry(Entry, Size);
+ if (Ret != OFFLOAD_SUCCESS)
+ REPORT() << "taskgraph could not release its mapping of ["
+ << Range.HostBegin << ", " << endOf(Range) << ")";
+}
+
+/// Drop the hold reference acquireRange took on one range, and remove the
+/// mapping if the graph created it and nothing in the graph asked for it to
+/// stay.
+void TaskGraphPlanner::releaseRange(DeviceTy &Device, TaskGraphRangeTy &Range) {
+ if (Range.StorageHeld) {
+ Range.StorageHeld = false;
+ Range.PlanInserted = false;
+ Device.getMappingInfo().removeTaskGraphClaim(
+ Range.HostBegin, static_cast<int64_t>(Range.Size));
+ if (Device.deleteData(Range.DevicePtr) != OFFLOAD_SUCCESS)
+ REPORT() << "taskgraph could not free its device memory for ["
+ << Range.HostBegin << ", " << endOf(Range) << ")";
+ return;
+ }
+ if (!Range.Held)
+ return;
+ Range.Held = false;
+
+ // A mapping the graph created and whose maps balance goes with the hold
+ // reference, which was its only reference.
+ releaseReference(Device, Range, /*UseHoldRefCount=*/true,
+ /*ForceDelete=*/false,
+ /*MayRemove=*/Range.PlanInserted && !Range.persists());
+
+ // `delete` acts on the dynamic count, which `ompx_hold` survives, so it is a
+ // separate release.
+ if (Range.Deleted) {
+ releaseReference(Device, Range, /*UseHoldRefCount=*/false,
+ /*ForceDelete=*/true, /*MayRemove=*/true);
+ Range.PlanInserted = false;
+ return;
+ }
+
+ // An unbalanced exit-data map drops a reference the graph did not take, on a
+ // mapping the graph found rather than made.
+ if (Range.releases() && !Range.PlanInserted)
+ releaseReference(Device, Range, /*UseHoldRefCount=*/false,
+ /*ForceDelete=*/false, /*MayRemove=*/true);
+
+ Range.PlanInserted = false;
+}
+
+/// Compute in \p TgtPteeBase the device address a pointer must hold, for an
+/// entry whose base is the pointer object and whose begin is its pointee, given
+/// the pointer's value \p HstPteeBase. Null if the pointee has no device copy
+/// this replay. Returns false if the pointee moved within its range, which the
+/// fixed grouping cannot follow.
+bool TaskGraphPlanner::pointeeDeviceBase(const TaskGraphRangeTy &Pointee,
+ int64_t PointeeOffset,
+ char *HstPteeBase, char *HstPteeBegin,
+ char *&TgtPteeBase) {
+ TgtPteeBase = nullptr;
+
+ if (HstPteeBegin != Pointee.begin() + PointeeOffset) {
+ REPORT() << "taskgraph replay found the pointee at " << HstPteeBegin
+ << " where the recording had it "
+ << static_cast<void *>(Pointee.begin() + PointeeOffset)
+ << ", inside the region [" << Pointee.HostBegin << ", "
+ << endOf(Pointee)
+ << "); a graph's mapped topology is fixed when it is recorded";
+ return false;
+ }
+ if (!Pointee.DevicePtr || Pointee.HostResident)
+ return true;
+
+ // The device pointer holds the device counterpart of the pointee's base, not
+ // of its begin: `map(s.p[10:n])` maps from `&s.p[10]`, but `s.p` points at
+ // `s.p[0]`.
+ TgtPteeBase = reinterpret_cast<char *>(Pointee.DevicePtr) + PointeeOffset -
+ (HstPteeBegin - HstPteeBase);
+ return true;
+}
+
+/// Fill in the bytes every attachment writes, for a replay about to start.
+/// Restaged every replay, since the pointee's device address depends on the
+/// current mapping state and on relocation.
+bool TaskGraphPlanner::stageTaskGraphAttachments(
+ DeviceTy &Device, TaskGraphTy &Graph, MutableArrayRef<uint8_t> Changes) {
+ TaskGraphDataPlanTy &Plan = Graph.Plan;
+ constexpr uint64_t VoidPtrSize = sizeof(void *);
+ MappingInfoTy &MappingInfo = Device.getMappingInfo();
+
+ for (TaskGraphNode *Node : Graph.leaves()) {
+ for (auto [I, S] : llvm::enumerate(argsOf(Node))) {
+ auto *A = dyn_cast<TaskGraphAttachArgTy>(&S.get());
+ if (!A)
+ continue;
+ A->Enabled = false;
+ const TaskGraphRangeTy &Pointer = *A->PtrRange;
+ // A pointer with no device copy of its own needs no correction.
+ if (!Pointer.DevicePtr || Pointer.HostResident)
+ continue;
+
+ const uint32_t Arg = static_cast<uint32_t>(I);
+ char *HstPtrAddr = argBase(Node, Arg);
+ char *HstPteeBegin = argBegin(Node, Arg);
+
+ char *TgtPteeBase = nullptr;
+ if (!pointeeDeviceBase(*A->PteeRange, A->PteeOffset,
+ *reinterpret_cast<char **>(HstPtrAddr),
+ HstPteeBegin, TgtPteeBase))
+ return false;
+ if (!TgtPteeBase)
+ continue;
+
+ char *Staging = Plan.AttachStaging.data() + A->StagingOffset;
+ std::memcpy(Plan.AttachHostBytes.data() + A->StagingOffset, HstPtrAddr,
+ A->Size);
+ // The rest of a Fortran descriptor is the same on the device.
+ uint8_t &Change = Changes[Plan.argIndex(S)];
+ auto stage = [&](uint64_t Offset, const void *Bytes, uint64_t Size) {
+ if (!std::memcmp(Staging + Offset, Bytes, Size))
+ return;
+ std::memcpy(Staging + Offset, Bytes, Size);
+ Change |= TaskGraphArgUpdateTy::DataChanged;
+ };
+ stage(0, &TgtPteeBase, VoidPtrSize);
+ if (A->Size > VoidPtrSize)
+ stage(VoidPtrSize, HstPtrAddr + VoidPtrSize, A->Size - VoidPtrSize);
+
+ // Record the shadow pointer on the pointer's mapping, as the runtime
+ // does, so that a `from` copy of the enclosing mapping does not hand the
+ // host a device address.
+ void *TgtPtrAddr =
+ reinterpret_cast<char *>(Pointer.DevicePtr) + A->Offset;
+ TargetPointerResultTy TPR = MappingInfo.getTgtPtrBegin(
+ HstPtrAddr, static_cast<int64_t>(A->Size), /*UpdateRefCount=*/false,
+ /*UseHoldRefCount=*/false, /*MustContain=*/true);
+ if (TPR.getEntry())
+ TPR.getEntry()->addShadowPointer(
+ ShadowPtrInfoTy{reinterpret_cast<void **>(HstPtrAddr),
+ reinterpret_cast<void **>(TgtPtrAddr), TgtPteeBase,
+ static_cast<int64_t>(A->Size)});
+
+ A->Enabled = true;
+ }
+ }
+ return true;
+}
+
+/// Take the initial value of every privatised argument, when a replay of the
+/// taskgraph begins, and mark those whose device copy is out of date.
+///
+/// A firstprivate is initialised from its variable as it stands when the
+/// replay begins (OpenMP 6.0 [14.3]), which can be well before the graph runs,
+/// except under `saved`, where it comes from the data environment saved at
+/// recording and so is staged once. Only bytes that changed are marked for
+/// copying.
+void TaskGraphPlanner::beginTaskGraphReplay(TaskGraphTy &Graph) {
+ TaskGraphDataPlanTy &Plan = Graph.Plan;
+
+ for (TaskGraphNode *Node : Graph.leaves()) {
+ for (auto [I, S] : llvm::enumerate(argsOf(Node))) {
+ const uint32_t Arg = static_cast<uint32_t>(I);
+ if (auto *F = dyn_cast<TaskGraphFirstPrivateArgTy>(&S.get())) {
+ if (F->Saved && F->Staged)
+ continue;
+
+ char *Staging = Plan.PrivateStaging.data() + F->StagingOffset;
+ const char *Bytes = argBegin(Node, Arg);
+
+ if (!F->Staged || std::memcmp(Staging, Bytes, F->Size)) {
+ std::memcpy(Staging, Bytes, F->Size);
+ F->InitPending = true;
+ }
+
+ F->Staged = true;
+ }
+ }
+ }
+}
+
+/// Put back the host value of every attached pointer whose range a copy brought
+/// back from the device, for a replay that has finished. The runtime does the
+/// same at the end of a data region. Pointers in ranges no copy brought back
+/// are left as the program has them.
+void TaskGraphPlanner::restoreTaskGraphAttachments(TaskGraphTy &Graph) {
+ TaskGraphDataPlanTy &Plan = Graph.Plan;
+ if (Plan.AttachStaging.empty())
+ return;
+
+ SmallVector<bool, 8> CopiedBack(Plan.Ranges.size(), false);
+ for (const TaskGraphArgSlotTy &S : Plan.Args) {
+ if (auto *A = dyn_cast<TaskGraphMappedArgTy>(&S.get())) {
+ if (A->Out.Requested && A->Out.Enabled)
+ CopiedBack[A->Range->Id] = true;
+ }
+ }
+
+ for (TaskGraphNode *Node : Graph.leaves())
+ for (auto [I, S] : llvm::enumerate(argsOf(Node)))
+ if (auto *A = dyn_cast<TaskGraphAttachArgTy>(&S.get()))
+ if (A->Enabled && CopiedBack[A->PtrRange->Id])
+ std::memcpy(argBase(Node, static_cast<uint32_t>(I)),
+ Plan.AttachHostBytes.data() + A->StagingOffset, A->Size);
+}
+
+/// Whether \p Node moves bytes into or out of the range numbered \p Range in
+/// the replay being acquired, whose enablement has been settled.
+bool TaskGraphPlanner::copiesThisReplay(TaskGraphNode *Node, uint32_t Range) {
+ for (const TaskGraphArgSlotTy &S : argsOf(Node)) {
+ if (const auto *A = dyn_cast<TaskGraphMappedArgTy>(&S.get())) {
+ if (A->Range->Id != Range)
+ continue;
+ if (A->In.Requested && A->In.Enabled)
+ return true;
+ if (A->Out.Requested && A->Out.Enabled)
+ return true;
+ } else if (const auto *T = dyn_cast<TaskGraphAttachArgTy>(&S.get())) {
+ if (T->PtrRange->Id != Range)
+ continue;
+ if (T->Enabled)
+ return true;
+ }
+ }
+ return false;
+}
+
+/// How many argument entries a leaf's plan has: one per recorded map argument,
+/// for the leaves that carry map arrays.
+static uint32_t numPlanArgs(const TaskGraphNode *Node) {
+ if (auto *TargetNode = dyn_cast<TaskGraphTargetNode>(Node)) {
+ auto *KA = static_cast<KernelArgsTy *>(TargetNode->KernelArgs);
+ return KA ? KA->NumArgs : 0;
+ }
+ if (auto *DataNode = dyn_cast<TaskGraphTargetDataNode>(Node))
+ return DataNode->ArgNum;
+ return 0;
+}
+
+/// Lay out every leaf's operations, in the order the leaf performs them, and
+/// every target leaf's kernel parameters. Returns false for a kernel argument
+/// that has no place in the plan, which should not happen.
+bool TaskGraphPlanner::buildOpsAndParams(TaskGraphTy &Graph) {
+ TaskGraphDataPlanTy &Plan = Graph.Plan;
+ using OpKindTy = TaskGraphOpTy::KindTy;
+ using ArgKindTy = TaskGraphArgBaseTy::KindTy;
+ using ParamKindTy = TaskGraphKernelParamTy::KindTy;
+
+ // Built end to end, and only then pointed at, since the vectors grow.
+ SmallVector<std::pair<size_t, size_t>, 16> OpSpans, ParamSpans;
+ for (TaskGraphNode *Node : Graph.leaves()) {
+ const size_t FirstOp = Plan.Ops.size();
+ const size_t FirstParam = Plan.KernelParams.size();
+ ArrayRef<TaskGraphArgSlotTy> Args = argsOf(Node);
+ auto each = [&](ArgKindTy Kind, auto &&Fn) {
+ for (auto [I, S] : llvm::enumerate(Args)) {
+ if (const auto *A = dyn_cast<TaskGraphArgBaseTy>(&S.get()))
+ if (A->Kind == Kind)
+ Fn(static_cast<uint32_t>(I), S);
+ }
+ };
+ auto copies = [&](bool In) {
+ each(ArgKindTy::Mapped, [&](uint32_t I, const TaskGraphArgSlotTy &S) {
+ const auto *M = cast<TaskGraphMappedArgTy>(&S.get());
+ if (In ? M->In.Requested : M->Out.Requested)
+ Plan.Ops.push_back({In ? OpKindTy::CopyIn : OpKindTy::CopyOut, I});
+ });
+ };
+ // Attachments follow the leaf's mapping and precede anything following the
+ // pointer; so does filling privatised storage, a private pointer
+ // initialised to a device address being an attachment in all but name.
+ auto attachments = [&]() {
+ each(ArgKindTy::Attach, [&](uint32_t I, const TaskGraphArgSlotTy &) {
+ Plan.Ops.push_back({OpKindTy::Attach, I});
+ });
+ each(ArgKindTy::FirstPrivate,
+ [&](uint32_t I, const TaskGraphArgSlotTy &) {
+ Plan.Ops.push_back({OpKindTy::PrivateInit, I});
+ });
+ };
+
+ if (isa<TaskGraphTargetDataNode>(Node)) {
+ each(ArgKindTy::Mapped, [&](uint32_t I, const TaskGraphArgSlotTy &S) {
+ const auto *M = cast<TaskGraphMappedArgTy>(&S.get());
+ if (M->In.Requested)
+ Plan.Ops.push_back({OpKindTy::CopyIn, I});
+ if (M->Out.Requested)
+ Plan.Ops.push_back({OpKindTy::CopyOut, I});
+ });
+ attachments();
+ } else {
+ auto *Target = cast<TaskGraphTargetNode>(Node);
+ copies(/*In=*/true);
+ attachments();
+ Plan.Ops.push_back({OpKindTy::Launch, 0});
+ copies(/*In=*/false);
+
+ // The host appends the launch environment after the last real target
+ // parameter, which finalize has checked the recording to be new enough
+ // to do. What remains are the kernel's parameters, in order.
+ auto *KA = static_cast<KernelArgsTy *>(Target->KernelArgs);
+ int32_t LaunchEnvironment = -1;
+ for (uint32_t I = 0; I < KA->NumArgs; ++I)
+ if (mt::isTargetParam(KA->ArgTypes[I]))
+ LaunchEnvironment = static_cast<int32_t>(I);
+ for (uint32_t I = 0; I < KA->NumArgs; ++I) {
+ if (!mt::isTargetParam(KA->ArgTypes[I]) ||
+ static_cast<int32_t>(I) == LaunchEnvironment)
+ continue;
+ const TaskGraphArgSlotTy &S = Args[I];
+ const TaskGraphArgBaseTy *A = &S.get();
+ switch (A->getKind()) {
+ case ArgKindTy::Literal:
+ Plan.KernelParams.push_back({ParamKindTy::Literal, I, 0});
+ break;
+ case ArgKindTy::FirstPrivate: {
+ const auto *F = cast<TaskGraphFirstPrivateArgTy>(A);
+ Plan.KernelParams.push_back({ParamKindTy::Private, I, F->BaseOffset});
+ break;
+ }
+ case ArgKindTy::Mapped: {
+ char *Base = static_cast<char *>(KA->ArgBasePtrs[I]);
+ const auto *M = cast<TaskGraphMappedArgTy>(A);
+ // A pointer-and-pointee entry records the address of the pointer as
+ // its base, and the kernel is handed what the pointer holds, as in
+ // targetKernel.
+ if (mt::isPtrAndObj(KA->ArgTypes[I]))
+ Base = *reinterpret_cast<char **>(Base);
+ Plan.KernelParams.push_back(
+ {ParamKindTy::Mapped, I,
+ static_cast<int64_t>(M->Offset) +
+ (Base - static_cast<char *>(KA->ArgPtrs[I]))});
+ break;
+ }
+ case ArgKindTy::None:
+ case ArgKindTy::Attach:
+ case ArgKindTy::NoTargetAttach:
+ assert(false && "kernel parameter with no place in the plan");
+ return false;
+ }
+ }
+ if (LaunchEnvironment != -1)
+ Plan.KernelParams.push_back({ParamKindTy::LaunchEnvironment, 0, 0});
+ }
+ OpSpans.push_back({FirstOp, Plan.Ops.size() - FirstOp});
+ ParamSpans.push_back({FirstParam, Plan.KernelParams.size() - FirstParam});
+ }
+
+ for (auto [Node, Ops, Params] :
+ llvm::zip_equal(Graph.leaves(), OpSpans, ParamSpans)) {
+ Node->OpPlan = Plan.Ops.data() + Ops.first;
+ Node->NumOps = Ops.second;
+ if (auto *Target = dyn_cast<TaskGraphTargetNode>(Node)) {
+ Target->Params = Plan.KernelParams.data() + Params.first;
+ Target->NumParams = Params.second;
+ }
+ }
+ return true;
+}
+
+bool TaskGraphPlanner::buildTaskGraphPlan(DeviceTy &Device,
+ TaskGraphTy &Graph) {
+ TaskGraphDataPlanTy &Plan = Graph.Plan;
+
+ // One entry per recorded argument of every leaf, in a single allocation that
+ // is never resized, so the leaves can point into it.
+ size_t NumArgs = 0;
+ for (TaskGraphNode *Node : Graph.leaves())
+ NumArgs += numPlanArgs(Node);
+ Plan.Args.assign(NumArgs, TaskGraphArgSlotTy{});
+ NumArgs = 0;
+ for (TaskGraphNode *Node : Graph.leaves()) {
+ Node->ArgPlan = Plan.Args.data() + NumArgs;
+ Node->NumArgPlans = numPlanArgs(Node);
+ NumArgs += Node->NumArgPlans;
+ }
+
+ // Classify every argument and collect the maps in encounter order, including
+ // maps that move no data: `alloc` and `release` still decide how long a
+ // range's mapping lives. Whether a range is mapped is decided per replay.
+ std::vector<ClaimTy> Claims;
+ // A graph the plan cannot express is replayed on the host. Say why, since
+ // it is otherwise silent. There is no taskgraph info category, so this
+ // shows under any of them.
+ auto decline = [&](uint32_t Leaf, uint32_t Arg, void *Begin,
+ const char *Why) {
+ INFO(OMP_INFOTYPE_ALL, Device.DeviceID,
+ "taskgraph %" PRIu64 " replays on the host: argument %" PRIu32
+ " of leaf %" PRIu32 " (" DPxMOD
+ ") %s, which the graph cannot lower onto the device\n",
+ static_cast<uint64_t>(Graph.GraphId), Arg, Leaf, DPxPTR(Begin), Why);
+ return false;
+ };
+ // An attachment's pointer and pointee are mapped by other maps, so both are
+ // looked up once the ranges are settled.
+ auto noteAttach = [&](TaskGraphArgSlotTy &S, uint64_t Size) {
+ auto &A = S.emplace<TaskGraphAttachArgTy>();
+ A.Kind = TaskGraphArgBaseTy::KindTy::Attach;
+ A.Size = Size;
+ A.StagingOffset = Plan.AttachStaging.size();
+ Plan.AttachStaging.resize(A.StagingOffset + Size);
+ Plan.AttachHostBytes.resize(A.StagingOffset + Size);
+ };
+ // A privatised argument gets a slot in the private block, laid out here and
+ // allocated once the whole graph has been walked. The staging bytes share
+ // the block's layout.
+ auto notePrivate = [&](TaskGraphArgSlotTy &S, void *HstPtrBegin,
+ void *HstPtrBase, uint64_t Size, bool Initialized,
+ bool IsSaved) {
+ uint64_t StagingOffset =
+ alignTo(Plan.PrivateStaging.size(), privateAlignment(HstPtrBegin));
+ uint64_t BaseOffset =
+ static_cast<char *>(HstPtrBase) - static_cast<char *>(HstPtrBegin);
+ if (Initialized) {
+ auto &F = S.emplace<TaskGraphFirstPrivateArgTy>();
+ F.Size = Size;
+ F.StagingOffset = StagingOffset;
+ F.BaseOffset = BaseOffset;
+ F.Saved = IsSaved;
+ }
+ Plan.PrivateStaging.resize(StagingOffset + Size);
+ Plan.PrivateHostBytes.resize(StagingOffset + Size);
+ return true;
+ };
+ auto noteClaim = [&](MutableArrayRef<TaskGraphArgSlotTy> Args, uint32_t I,
+ void *Begin, uint64_t Size, int64_t ArgType,
+ ClaimKindTy Kind) -> TaskGraphMappedArgTy & {
+ TaskGraphArgSlotTy &S = Args[I];
+ // An entry of no extent, such as `map(p[0:0])`, only looks memory up and
+ // takes no part in the reference count.
+ if (!Size)
+ Kind = ClaimKindTy::Lookup;
+ auto &M = S.emplace<TaskGraphMappedArgTy>();
+ M.Size = Size;
+ // A struct's entry precedes its members', so it has been classified.
+ const TaskGraphArgSlotTy *Parent = nullptr;
+ const int64_t P = structParentIndex(ArgType);
+ if (Kind != ClaimKindTy::Lookup && P >= 0 && P < I &&
+ isa<TaskGraphMappedArgTy>(Args[P].get()))
+ Parent = &Args[P];
+ Claims.push_back(
+ ClaimTy{Kind, &S, static_cast<char *>(Begin), Size, ArgType, Parent});
+ return M;
+ };
+ // A map's copies. `alloc`, `release` and `delete` move no bytes.
+ auto noteCopies = [](TaskGraphMappedArgTy &M, mt::TransferDir Dir) {
+ if (!M.Size || Dir == mt::TransferDir::None)
+ return;
+ M.In.Requested =
+ Dir == mt::TransferDir::H2D || Dir == mt::TransferDir::Both;
+ M.Out.Requested =
+ Dir == mt::TransferDir::D2H || Dir == mt::TransferDir::Both;
+ };
+
+ for (uint32_t L = 0; L < Graph.leaves().size(); ++L) {
+ TaskGraphNode *Node = Graph.leaves()[L];
+ MutableArrayRef<TaskGraphArgSlotTy> Args = argsOf(Node);
+
+ if (auto *DataNode = dyn_cast<TaskGraphTargetDataNode>(Node)) {
+ // `target update` transfers whoever mapped the region; enter and exit
+ // maps transfer only on the reference-count change that owns the copy.
+ const bool IsUpdate = isa<TaskGraphTargetUpdateDataNode>(DataNode);
+ const bool IsEnter = isa<TaskGraphTargetEnterDataNode>(DataNode);
+ const bool IsExit = isa<TaskGraphTargetExitDataNode>(DataNode);
+ for (int32_t I = 0; I < DataNode->ArgNum; ++I) {
+ TaskGraphArgSlotTy &S = Args[I];
+ const int64_t ArgType = DataNode->ArgTypes[I];
+ // A data leaf has no kernel to hand privatised storage to, so the
+ // runtime's data paths skip these entries too.
+ if (mt::isLiteral(ArgType) || isPrivatised(ArgType))
+ continue;
+ if (isAttachment(ArgType)) {
+ noteAttach(S, static_cast<uint64_t>(DataNode->ArgSizes[I]));
+ continue;
+ }
+ if (const char *Why = whyUnlowerable(
+ ArgType,
+ DataNode->ArgMappers ? DataNode->ArgMappers[I] : nullptr))
+ return decline(L, I, DataNode->Args[I], Why);
+ auto &M =
+ noteClaim(Args, I, DataNode->Args[I],
+ static_cast<uint64_t>(DataNode->ArgSizes[I]), ArgType,
+ IsEnter ? ClaimKindTy::Enter
+ : IsExit ? ClaimKindTy::Exit
+ : ClaimKindTy::Lookup);
+ noteCopies(M, directionFor(DataNode, ArgType));
+ M.Always = IsUpdate || mt::isAlways(ArgType);
+ }
+ continue;
+ }
+
+ auto *TargetNode = cast<TaskGraphTargetNode>(Node);
+ auto *KA = static_cast<KernelArgsTy *>(TargetNode->KernelArgs);
+ if (!KA)
+ return false;
+
+ for (uint32_t I = 0; I < KA->NumArgs; ++I) {
+ TaskGraphArgSlotTy &S = Args[I];
+ const int64_t ArgType = KA->ArgTypes[I];
+ if (isLambdaCaptureFixup(ArgType))
+ return decline(L, I, KA->ArgPtrs[I],
+ "is a lambda's by-reference capture");
+ if (mt::isLiteral(ArgType)) {
+ // Pass-by-value: the argument is the value, not an address.
+ if (mt::isTargetParam(ArgType)) {
+ uint64_t Val = reinterpret_cast<uint64_t>(KA->ArgBasePtrs[I]);
+ // Apparently only 'Size' bytes of the literal are valid data, so
+ // we must mask the rest off.)
+ uint64_t BitSize = KA->ArgSizes[I] * 8;
+ assert(BitSize <= 64);
+ uint64_t Mask =
+ BitSize == 64 ? ~(uint64_t)0 : (((uint64_t)1 << BitSize) - 1);
+ S.emplace<TaskGraphLiteralArgTy>(Val & Mask);
+ }
+ continue;
+ }
+ if (isPrivatised(ArgType)) {
+ // We aren't expecting to see a 'PRIVATE | ATTACH' entry from current
+ // Clang. If we do see one, decline: if we need this later for some
+ // reason, we can add a new argument type for it.
+ if (mt::isAttach(ArgType))
+ return decline(L, I, KA->ArgPtrs[I],
+ "unexpected mapping-bit encoding 'PRIVATE | ATTACH'");
+ notePrivate(S, /*HstPtrBegin=*/KA->ArgPtrs[I],
+ /*HstPtrBase=*/KA->ArgBasePtrs[I],
+ static_cast<uint64_t>(KA->ArgSizes[I]),
+ /*Initialized=*/mt::isTo(ArgType), mt::isSaved(ArgType));
+ continue;
+ }
+ if (isAttachment(ArgType)) {
+ noteAttach(S, static_cast<uint64_t>(KA->ArgSizes[I]));
+ continue;
+ }
+ if (const char *Why = whyUnlowerable(
+ ArgType, KA->ArgMappers ? KA->ArgMappers[I] : nullptr))
+ return decline(L, I, KA->ArgPtrs[I], Why);
+
+ // A `target` region's map balances within the region, but may create
+ // the mapping.
+ auto &M = noteClaim(Args, I, KA->ArgPtrs[I],
+ static_cast<uint64_t>(KA->ArgSizes[I]), ArgType,
+ ClaimKindTy::Balanced);
+ noteCopies(M, mt::transferDir(ArgType));
+ M.Always = mt::isAlways(ArgType);
+ }
+ }
+
+ // Gather the maps into ranges, then lay the ranges out in address order,
+ // which numbers them.
+ RangeMapTy Gathered;
+ for (const ClaimTy &C : Claims)
+ gatherClaim(Gathered, C);
+ Plan.Ranges.reserve(Gathered.size());
+ for (auto &[Key, Range] : Gathered) {
+ Range.Id = Plan.Ranges.size();
+ Plan.Ranges.push_back(Range);
+ }
+
+ // Place every argument in the range holding it. A map always finds one;
+ // a lookup finds one only if some map in the graph covers the memory.
+ for (TaskGraphNode *Node : Graph.leaves()) {
+ for (auto [I, S] : llvm::enumerate(argsOf(Node))) {
+ const uint32_t Arg = static_cast<uint32_t>(I);
+ if (auto *M = dyn_cast<TaskGraphMappedArgTy>(&S.get())) {
+ char *Begin = argBegin(Node, Arg);
+ M->Range = containingRange(Plan.Ranges, Begin, M->Size);
+ assert(M->Range && "a gathered map outside every range");
+ M->Offset = Begin - M->Range->begin();
+ } else if (auto *A = dyn_cast<TaskGraphAttachArgTy>(&S.get())) {
+ // The pointee is looked up with zero extent, since its own map says
+ // how much of it there is.
+ char *Pointer = argBase(Node, Arg);
+ char *Pointee = argBegin(Node, Arg);
+ A->PtrRange = containingRange(Plan.Ranges, Pointer, A->Size);
+ A->PteeRange = containingRange(Plan.Ranges, Pointee, 0);
+ // An attachment missing either end is dropped, as the runtime drops it
+ // when a lookup comes back absent. The usual case is `map(p[0:n])` on
+ // a plain pointer, where `p` is privatised and has no device copy.
+ if (!A->PtrRange || !A->PteeRange) {
+ S.emplace<TaskGraphNoTargetAttachArgTy>();
+ } else {
+ A->Offset = Pointer - A->PtrRange->begin();
+ A->PteeOffset = Pointee - A->PteeRange->begin();
+ }
+ }
+ }
+ }
+
+ if (!buildOpsAndParams(Graph)) {
+ INFO(OMP_INFOTYPE_ALL, Device.DeviceID,
+ "taskgraph %" PRIu64 " replays on the host: a kernel argument has no "
+ "place in its plan\n",
+ static_cast<uint64_t>(Graph.GraphId));
+ return false;
+ }
+
+ // Decide which map over each range performs its implicit copies. Outside a
+ // graph a map copies in on the reference-count increment that creates the
+ // mapping and out on the decrement that removes it, so `target enter data
+ // map(alloc:)` followed by a `target` region mapping the same data `tofrom`
+ // copies in neither direction. Walk the recorded maps in order with a
+ // simulated count starting at zero; a range the graph did not create is
+ // borrowed, and its implicit copies are switched off per replay instead --
+ // except the copy out of an exit that drops the borrowed mapping's last
+ // reference, which a replay can only tell once it finds the mapping.
+ SmallVector<int32_t, 8> Count(Plan.Ranges.size(), 0);
+ SmallVector<bool, 8> Borrowed(Plan.Ranges.size(), false);
+ for (const ClaimTy &C : Claims) {
+ auto *Arg = cast<TaskGraphMappedArgTy>(&C.Arg->get());
+ // A struct member copies on its struct's count change and makes none of
+ // its own. The runtime processes an exit's members before their struct,
+ // but that is the same decrement to zero.
+ if (C.Parent) {
+ auto *Parent = cast<TaskGraphMappedArgTy>(&C.Parent->get());
+ Arg->In.Perimeter = Parent->In.Perimeter;
+ Arg->Out.Perimeter = Parent->Out.Perimeter;
+ Arg->OwnsBorrowedOut = Parent->OwnsBorrowedOut;
+ continue;
+ }
+ int32_t &N = Count[Arg->Range->Id];
+ switch (C.Kind) {
+ case ClaimKindTy::Balanced:
+ // Entry and exit of one region, with nothing in between.
+ if (++N == 1)
+ Arg->In.Perimeter = true;
+ if (--N == 0)
+ Arg->Out.Perimeter = true;
+ break;
+ case ClaimKindTy::Enter:
+ if (++N == 1)
+ Arg->In.Perimeter = true;
+ break;
+ case ClaimKindTy::Exit:
+ // The copy-out happens on the decrement to zero.
+ if (N == 1)
+ Arg->Out.Perimeter = true;
+ else if (N == 0 && !Borrowed[Arg->Range->Id])
+ Arg->OwnsBorrowedOut = Borrowed[Arg->Range->Id] = true;
+ N = std::max(0, N - 1);
+ break;
+ case ClaimKindTy::Lookup:
+ break;
+ }
+ }
+
+ // Outside a graph, whichever construct maps a range first copies it, and a
+ // construct that finds the range mapped waits for that copy to land, so a
+ // map is atomic as OpenMP requires. Here the walk above settled who copies
+ // in leaf order, and only the region tree orders one leaf against another:
+ // a leaf the tree leaves unordered with a range's copier could use the range
+ // mid-copy, or before it. Record each such pair, so that a replay in which
+ // the copy does run is declined to the host rather than ordered. A replay
+ // that finds the range mapped outside the graph copies nothing, and keeps the
+ // device graph.
+ //
+ // Two leaves that each copy the range in for themselves and write nothing
+ // back are exempt: each kernel follows its own copy, and the other's copy
+ // writes the same bytes, since the host data cannot change meanwhile without
+ // a race in the program. That is a shared input read by concurrent tasks.
+ {
+ struct UseTy {
+ uint32_t Leaf;
+ /// Some argument of the leaf moves bytes into or out of the range.
+ bool Copies = false;
+ /// It copies the range in itself, as the owner of the copy.
+ bool OwnsIn = false;
+ /// It moves bytes some other way: a copy out, an attachment, or a copy
+ /// regardless of ownership.
+ bool Writes = false;
+ };
+ std::vector<SmallVector<UseTy, 2>> Uses(Plan.Ranges.size());
+ for (uint32_t L = 0; L < Graph.leaves().size(); ++L)
+ for (const TaskGraphArgSlotTy &S : argsOf(Graph.leaves()[L])) {
+ if (const auto *A = dyn_cast<TaskGraphAttachArgTy>(&S.get())) {
+ SmallVector<UseTy, 2> &RangeUses = Uses[A->PtrRange->Id];
+ if (RangeUses.empty() || RangeUses.back().Leaf != L)
+ RangeUses.push_back({L});
+ UseTy &Use = RangeUses.back();
+ // An attachment's write is unconditional.
+ Use.Copies = true;
+ Use.Writes = true;
+ } else if (const auto *M = dyn_cast<TaskGraphMappedArgTy>(&S.get())) {
+ SmallVector<UseTy, 2> &RangeUses = Uses[M->Range->Id];
+ if (RangeUses.empty() || RangeUses.back().Leaf != L)
+ RangeUses.push_back({L});
+ UseTy &Use = RangeUses.back();
+ const bool Forced =
+ (M->In.Requested || M->Out.Requested) && M->Always;
+ const bool OwnsIn = M->In.Requested && M->In.Perimeter;
+ const bool OwnsOut =
+ M->Out.Requested && (M->Out.Perimeter || M->OwnsBorrowedOut);
+ Use.Copies |= Forced || OwnsIn || OwnsOut;
+ Use.OwnsIn |= OwnsIn;
+ Use.Writes |= Forced || OwnsOut;
+ }
+ }
+ auto readsOwnCopy = [](const UseTy &Use) {
+ return Use.OwnsIn && !Use.Writes;
+ };
+
+ for (const TaskGraphRangeTy &Range : Plan.Ranges) {
+ ArrayRef<UseTy> RangeUses = Uses[Range.Id];
+ for (const UseTy &Copier : RangeUses) {
+ if (!Copier.Copies)
+ continue;
+ // Every copier gets its own entry, since a replay may run the copies of
+ // one and not the other's.
+ for (const UseTy &Other : RangeUses) {
+ if (Other.Leaf == Copier.Leaf)
+ continue;
+ if ((readsOwnCopy(Copier) && readsOwnCopy(Other)) ||
+ areOrdered(*Graph.leaves()[Copier.Leaf],
+ *Graph.leaves()[Other.Leaf]))
+ continue;
+ // One unordered user is enough to name.
+ Plan.CopyConflicts.push_back({Range.Id, Copier.Leaf, Other.Leaf});
+ break;
+ }
+ }
+ }
+ }
+
+ // Decide which ranges the graph owns outright. A range the graph creates,
+ // whose maps balance, and that nothing outside the graph has mapped needs no
+ // mapping-table entry, so each replay gives it storage of its own. The
+ // decision is fixed here, against the mapping state the recording left. A
+ // range that is only partly mapped declines the graph. Device addresses stay
+ // null until the first replay resolves them.
+ MappingInfoTy &MappingInfo = Device.getMappingInfo();
+ {
+ auto HDTTMap = MappingInfo.HostDataToTargetMap.getExclusiveAccessor();
+ for (TaskGraphRangeTy &Range : Plan.Ranges) {
+ if (!Range.Size)
+ continue;
+ LookupResult LR = MappingInfo.lookupMapping(
+ HDTTMap, Range.HostBegin, static_cast<int64_t>(Range.Size));
+ const bool Mapped = LR.Flags.IsContained;
+ const bool Partial = LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter;
+ LR.TPR.reset();
+ if (Partial) {
+ MESSAGE("taskgraph %" PRIu64 " replays on the host: [" DPxMOD
+ ", " DPxMOD ") is only partly mapped outside the graph",
+ static_cast<uint64_t>(Graph.GraphId), DPxPTR(Range.HostBegin),
+ DPxPTR(Range.end()));
+ return false;
+ }
+ Range.SelfContained = !Mapped && Range.MayCreate && !Range.RefDelta &&
+ !Range.Deleted &&
+ !hostResidentIfUnmapped(Device, Range);
+ }
+ }
+
+ // A device pointer that outlives the replay must not point into storage that
+ // does not, so an attachment whose pointer is not self-contained demotes its
+ // pointee. That can demote the pointer of another attachment, hence the
+ // fixpoint.
+ for (bool Changed = true; Changed;) {
+ Changed = false;
+ for (TaskGraphArgSlotTy &S : Plan.Args)
+ if (auto *A = dyn_cast<TaskGraphAttachArgTy>(&S.get()))
+ if (A->PteeRange->SelfContained && !A->PtrRange->SelfContained) {
+ A->PteeRange->SelfContained = false;
+ Changed = true;
+ }
+ }
+
+ Plan.Epoch = MappingInfo.getEpoch();
+
+ // Private storage is not in the device data environment, so it is allocated
+ // once for the graph, at an address lowered kernel arguments can name.
+ if (!Plan.PrivateStaging.empty()) {
+ Plan.PrivateBlock = Device.allocData(Plan.PrivateStaging.size());
+ if (!Plan.PrivateBlock) {
+ REPORT() << "taskgraph could not allocate " << Plan.PrivateStaging.size()
+ << " bytes of device memory for its privatised arguments";
+ return false;
+ }
+ }
+ return true;
+}
+
+bool TaskGraphPlanner::acquireTaskGraphMappings(
+ DeviceTy &Device, TaskGraphTy &Graph,
+ SmallVectorImpl<TaskGraphArgUpdateTy> &Updates, bool &Resync) {
+ Updates.clear();
+ TaskGraphDataPlanTy &Plan = Graph.Plan;
+ // libomp holds the record's map_lock for the whole of a replay, so a graph is
+ // never acquired twice at once.
+ assert(!Plan.MappingsHeld && "taskgraph replay entered twice");
+ Resync = Plan.Resync;
+ Plan.Resync = true;
+ Plan.DeclineReported = false;
+
+ // Relocation has already rewritten the recorded maps, so move the ranges
+ // before resolving them against the mapping table.
+ SmallVector<bool, 8> HostMoved(Plan.Ranges.size(), false);
+ if (!rekeyRanges(Graph, HostMoved))
+ return false;
+
+ // The epoch baseline is taken when the previous replay released its
+ // mappings, so the graph's own mapping and unmapping do not count as drift.
+ const bool Moved = llvm::is_contained(HostMoved, true);
+ if (reportDrift() &&
+ (Moved || Device.getMappingInfo().getEpoch() != Plan.Epoch))
+ fprintf(stderr, "taskgraph: %s, re-resolving ranges\n",
+ Moved ? "relocated host captures" : "mapping drift");
+
+ // Set before acquiring anything, so that the ranges acquired before a failure
+ // are released.
+ Plan.MappingsHeld = true;
+
+ SmallVector<bool, 8> DeviceMoved(Plan.Ranges.size(), false);
+ SmallVector<uint8_t, 16> Changes(Plan.Args.size(), 0);
+
+ for (TaskGraphRangeTy &Range : Plan.Ranges) {
+ void *Address = nullptr;
+ bool Inserted = false;
+ bool HostResident = false;
+ if (!acquireRange(Device, Range, Address, Inserted, HostResident))
+ return false;
+
+ DeviceMoved[Range.Id] = Range.DevicePtr != Address;
+ Range.DevicePtr = Address;
+ Range.PlanInserted = Inserted;
+ Range.HostResident = HostResident;
+ }
+
+ // Borrowing a self-contained range can recreate the shape the build ruled
+ // out: an attachment writing into a mapping that outlives the replay the
+ // address of storage that does not. Decline such a replay.
+ for (const TaskGraphArgSlotTy &S : Plan.Args)
+ if (auto *A = dyn_cast<TaskGraphAttachArgTy>(&S.get()))
+ if (A->PteeRange && A->PtrRange->SelfContained && A->PtrRange->Held &&
+ A->PteeRange->StorageHeld) {
+ MESSAGE("taskgraph borrows [" DPxMOD ", " DPxMOD
+ ") from a mapping outside the graph, which would be left "
+ "pointing at device memory only this replay keeps",
+ DPxPTR(A->PtrRange->HostBegin), DPxPTR(A->PtrRange->end()));
+ return false;
+ }
+
+ // An implicit transfer runs only if the graph created this replay's mapping
+ // (PlanInserted) and this map is the one whose count change performs it
+ // (OwnsIn or OwnsOut), or the graph removes a mapping it found (DropsLast)
+ // and this is the exit that drops it (OwnsBorrowedOut). `always` and
+ // `target update` transfer regardless. A host-resident range has nothing to
+ // copy. A toggle is reported when the resulting enablement changes, not
+ // when its inputs do.
+ for (TaskGraphArgSlotTy &S : Plan.Args) {
+ if (auto *M = dyn_cast<TaskGraphMappedArgTy>(&S.get())) {
+ const TaskGraphRangeTy &Range = *M->Range;
+ const bool Resolved = !Range.HostResident && Range.DevicePtr;
+ auto enable = [&](bool Copies, bool Owns, bool &Enabled) {
+ if (!Copies)
+ return;
+ const bool Was = Enabled;
+ Enabled = Resolved && (Owns || M->Always);
+ if (Enabled != Was)
+ Changes[Plan.argIndex(S)] |= TaskGraphArgUpdateTy::EnabledChanged;
+ };
+ enable(M->In.Requested, Range.PlanInserted && M->In.Perimeter,
+ M->In.Enabled);
+ enable(M->Out.Requested,
+ (Range.PlanInserted && M->Out.Perimeter) ||
+ (Range.DropsLast && M->OwnsBorrowedOut),
+ M->Out.Enabled);
+ }
+ }
+
+ // An attachment depends on its pointee's range as well as its pointer's, so
+ // its enablement is compared after staging.
+ SmallVector<bool, 8> WasAttached(Plan.Args.size());
+ for (uint32_t I = 0; I < Plan.Args.size(); ++I)
+ if (const auto *A = dyn_cast<TaskGraphAttachArgTy>(&Plan.Args[I].get()))
+ WasAttached[I] = A->Enabled;
+
+ if (!stageTaskGraphAttachments(Device, Graph, Changes))
+ return false;
+
+ // A range a leaf copies while a leaf nothing orders with it uses the range
+ // has to go to the host, but only when the copy runs: a range mapped outside
+ // the graph has none of its implicit copies this replay.
+ for (const TaskGraphDataPlanTy::CopyConflictTy &C : Plan.CopyConflicts) {
+ if (!copiesThisReplay(Graph.leaves()[C.Copier], C.Range))
+ continue;
+ // Reported when it starts happening, not on every replay it goes on for,
+ // and only under LIBOMPTARGET_INFO: unlike the other declines here, this
+ // one is the plan working as intended.
+ Plan.DeclineReported = true;
+ if (!Plan.ConflictDeclined) {
+ const TaskGraphRangeTy &Range = Plan.Ranges[C.Range];
+ INFO(OMP_INFOTYPE_ALL, Device.DeviceID,
+ "taskgraph %" PRIu64 " replays on the host: leaf %" PRIu32
+ " copies [" DPxMOD ", " DPxMOD ") and leaf %" PRIu32
+ " uses it, and nothing orders the two\n",
+ static_cast<uint64_t>(Graph.GraphId), C.Copier,
+ DPxPTR(Range.HostBegin), DPxPTR(Range.end()), C.User);
+ }
+ Plan.ConflictDeclined = true;
+ return false;
+ }
+ Plan.ConflictDeclined = false;
+
+ for (TaskGraphNode *Node : Graph.leaves()) {
+ for (auto [I, Slot] : llvm::enumerate(argsOf(Node))) {
+ auto *Arg = &Slot.get();
+ uint8_t &Change = Changes[Plan.argIndex(Slot)];
+ if (const auto *M = dyn_cast<TaskGraphMappedArgTy>(Arg)) {
+ if (DeviceMoved[M->Range->Id])
+ Change |= TaskGraphArgUpdateTy::DeviceAddressChanged;
+ if (HostMoved[M->Range->Id])
+ Change |= TaskGraphArgUpdateTy::HostAddressChanged;
+ } else if (const auto *A = dyn_cast<TaskGraphAttachArgTy>(Arg)) {
+ if (DeviceMoved[A->PtrRange->Id])
+ Change |= TaskGraphArgUpdateTy::DeviceAddressChanged;
+ if (A->Enabled != WasAttached[Plan.argIndex(Slot)])
+ Change |= TaskGraphArgUpdateTy::EnabledChanged;
+ } else if (const auto *F = dyn_cast<TaskGraphFirstPrivateArgTy>(Arg)) {
+ if (F->InitPending)
+ Change |= TaskGraphArgUpdateTy::DataChanged;
+ } else if (auto *L = dyn_cast<TaskGraphLiteralArgTy>(Arg)) {
+ KernelArgsTy *KernArgs = static_cast<KernelArgsTy *>(
+ cast<TaskGraphTargetNode>(Node)->KernelArgs);
+ // Relocation has re-read the variable into the recorded slot.
+ uint64_t Value = reinterpret_cast<uint64_t>(KernArgs->ArgBasePtrs[I]);
+ uint64_t BitSize = KernArgs->ArgSizes[I] * 8;
+ assert(BitSize <= 64);
+ uint64_t Mask =
+ BitSize == 64 ? ~(uint64_t)0 : (((uint64_t)1 << BitSize) - 1);
+ Value &= Mask;
+ if (Value != L->LiteralValue)
+ Change |= TaskGraphArgUpdateTy::DataChanged;
+ L->LiteralValue = Value;
+ }
+ }
+ }
+
+ // Report only the entries that changed. A resync makes the backend re-read
+ // everything, so it needs no list.
+ for (auto [Index, Change] : llvm::enumerate(Changes))
+ if (Change)
+ Updates.push_back({static_cast<uint32_t>(Index), Change});
+ return true;
+}
+
+void TaskGraphPlanner::commitTaskGraphUpdates(TaskGraphTy &Graph) {
+ Graph.Plan.Resync = false;
+}
+
+void TaskGraphPlanner::releaseTaskGraphMappings(DeviceTy &Device,
+ TaskGraphTy &Graph,
+ bool Replayed) {
+ if (!Graph.Plan.MappingsHeld)
+ return;
+ Graph.Plan.MappingsHeld = false;
+ // A declined replay may not have reached staging, so it has neither host
+ // bytes to restore nor private copies the backend has performed.
+ if (Replayed) {
+ restoreTaskGraphAttachments(Graph);
+ for (TaskGraphArgSlotTy &S : Graph.Plan.Args)
+ if (auto *A = dyn_cast<TaskGraphFirstPrivateArgTy>(&S.get()))
+ A->InitPending = false;
+ }
+ for (TaskGraphRangeTy &Range : Graph.Plan.Ranges)
+ releaseRange(Device, Range);
+ // Taken after the graph's own unmapping, so only the program's changes count
+ // as drift.
+ Graph.Plan.Epoch = Device.getMappingInfo().getEpoch();
+}
+
+void TaskGraphPlanner::releaseTaskGraphPlan(DeviceTy &Device,
+ TaskGraphTy &Graph) {
+ // A graph is never destroyed mid-replay, but hold references left behind
+ // could never be released, so do not rely on it.
+ releaseTaskGraphMappings(Device, Graph);
+ for (TaskGraphRangeTy &Range : Graph.Plan.Ranges)
+ Range.DevicePtr = nullptr;
+
+ if (Graph.Plan.PrivateBlock) {
+ if (Device.deleteData(Graph.Plan.PrivateBlock) != OFFLOAD_SUCCESS)
+ REPORT() << "taskgraph could not release the device memory holding its "
+ "privatised arguments";
+ Graph.Plan.PrivateBlock = nullptr;
+ }
+}
+
+bool llvm::omp::target::buildTaskGraphPlan(DeviceTy &Device,
+ TaskGraphTy &Graph) {
+ return TaskGraphPlanner::buildTaskGraphPlan(Device, Graph);
+}
+
+bool llvm::omp::target::acquireTaskGraphMappings(
+ DeviceTy &Device, TaskGraphTy &Graph,
+ SmallVectorImpl<TaskGraphArgUpdateTy> &Updates, bool &Resync) {
+ return TaskGraphPlanner::acquireTaskGraphMappings(Device, Graph, Updates,
+ Resync);
+}
+
+void llvm::omp::target::commitTaskGraphUpdates(TaskGraphTy &Graph) {
+ TaskGraphPlanner::commitTaskGraphUpdates(Graph);
+}
+
+void llvm::omp::target::releaseTaskGraphMappings(DeviceTy &Device,
+ TaskGraphTy &Graph,
+ bool Replayed) {
+ TaskGraphPlanner::releaseTaskGraphMappings(Device, Graph, Replayed);
+}
+
+void llvm::omp::target::beginTaskGraphReplay(TaskGraphTy &Graph) {
+ TaskGraphPlanner::beginTaskGraphReplay(Graph);
+}
+
+void llvm::omp::target::releaseTaskGraphPlan(DeviceTy &Device,
+ TaskGraphTy &Graph) {
+ TaskGraphPlanner::releaseTaskGraphPlan(Device, Graph);
+}
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index 4b65077f112400..57ebd332aea580 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -16,6 +16,7 @@
#include "OpenMP/Mapping.h"
#include "OpenMP/OMPT/Callback.h"
#include "OpenMP/OffloadRTL.h"
+#include "OpenMP/TaskGraphPlan.h"
#include "OpenMP/omp.h"
#include "PluginManager.h"
#include "device.h"
@@ -697,11 +698,17 @@ EXTERN void *__tgt_get_mapped_ptr(int64_t DeviceId, const void *HostPtr) {
// block of that size.
//===----------------------------------------------------------------------===//
+using llvm::omp::target::acquireTaskGraphMappings;
+using llvm::omp::target::buildTaskGraphPlan;
+using llvm::omp::target::commitTaskGraphUpdates;
+using llvm::omp::target::releaseTaskGraphPlan;
+using llvm::omp::target::TaskGraphArgUpdateTy;
using llvm::omp::target::TaskGraphBuilder;
using llvm::omp::target::TaskGraphEdge;
using llvm::omp::target::TaskGraphElement;
using llvm::omp::target::TaskGraphExclRegion;
using llvm::omp::target::TaskGraphIrreducibleRegion;
+using llvm::omp::target::TaskGraphMappingsGuard;
using llvm::omp::target::TaskGraphNode;
using llvm::omp::target::TaskGraphParRegion;
using llvm::omp::target::TaskGraphRegion;
@@ -914,6 +921,11 @@ EXTERN int __tgt_taskgraph_finalize(void *Graph) {
}
DeviceTy &Device = *DeviceOrErr;
+ // Building the plan allocates device memory, which is wasted on a device
+ // with no graph backend.
+ if (!Device.RTL->supports_taskgraphs(Device.RTLDeviceID))
+ return OFFLOAD_FAIL;
+
if (const char *Why = whyNotOffloadable(*G, ResolvedDevice)) {
INFO(OMP_INFOTYPE_ALL, Device.DeviceID,
"taskgraph %" PRIu64 " replays on the host: %s\n",
@@ -923,23 +935,32 @@ EXTERN int __tgt_taskgraph_finalize(void *Graph) {
G->Services.resolveKernel = taskGraphResolveKernel;
+ // Work out where the graph's mapped data lives before offering it to the
+ // plugin. A graph the plan cannot express is replayed on the host.
+ if (!buildTaskGraphPlan(Device, *G)) {
+ releaseTaskGraphPlan(Device, *G);
+ return OFFLOAD_FAIL;
+ }
+
// Offer the tree to the plugin, which lowers it however suits its backend.
// The base plugin implementation declines, as does a backend that meets
// something in the graph it cannot express; either way libomp keeps the graph
// and replays it on the host.
void *PluginGraph = nullptr;
if (Device.RTL->finalize_taskgraph(Device.RTLDeviceID, G, &PluginGraph) !=
- OFFLOAD_SUCCESS)
+ OFFLOAD_SUCCESS) {
+ releaseTaskGraphPlan(Device, *G);
return OFFLOAD_FAIL;
+ }
assert(PluginGraph && "plugin claimed a taskgraph without a handle");
G->PluginGraph = PluginGraph;
return OFFLOAD_SUCCESS;
}
EXTERN void __tgt_taskgraph_begin_replay(void *Graph) {
- // Nothing a graph holds yet depends on when the replay started.
- assert(static_cast<TaskGraphTy *>(Graph)->PluginGraph &&
- "replaying a graph no plugin claimed");
+ auto *G = static_cast<TaskGraphTy *>(Graph);
+ assert(G->PluginGraph && "replaying a graph no plugin claimed");
+ beginTaskGraphReplay(*G);
}
EXTERN int __tgt_taskgraph_replay(void *Graph) {
@@ -951,10 +972,57 @@ EXTERN int __tgt_taskgraph_replay(void *Graph) {
FATAL_MESSAGE(G->DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());
DeviceTy &Device = *DeviceOrErr;
- // The backend issues the graph and waits for it. Part of the graph may have
- // run by the time it fails, so a failure cannot fall back to the host.
- if (Device.RTL->replay_taskgraph(Device.RTLDeviceID, G) != OFFLOAD_SUCCESS)
+ // Resolve every range against the mapping table as it stands now, mapping
+ // the ones the graph is responsible for and holding all of them until the
+ // graph has run, then hand the plugin whatever changed.
+ //
+ // Until the handover nothing has been issued, so a graph that cannot be run
+ // is declined: libomp runs its target constructs itself instead. The guard
+ // gives back whatever was acquired.
+ auto decline = [&](const char *Why) {
+ if (Why)
+ MESSAGE("taskgraph %" PRIu64 " replays on the host this time: %s",
+ static_cast<uint64_t>(G->GraphId), Why);
+ return OFFLOAD_FAIL;
+ };
+ llvm::SmallVector<TaskGraphArgUpdateTy, 8> Updates;
+ bool Resync = false;
+ TaskGraphMappingsGuard Mappings(Device, *G);
+ if (!acquireTaskGraphMappings(Device, *G, Updates, Resync))
+ return decline(G->Plan.declineReported()
+ ? nullptr
+ : "it cannot be replayed against the current mapping "
+ "state");
+ // Pinned every replay, since the ranges' host memory is only known to be the
+ // program's while the graph runs. A failed pin holds nothing.
+ if (Device.RTL->pin_taskgraph(Device.RTLDeviceID, G) != OFFLOAD_SUCCESS)
+ return decline("the device could not pin the graph's host memory");
+ if (llvm::any_of(Updates, [&](const TaskGraphArgUpdateTy &U) {
+ return U.has(TaskGraphArgUpdateTy::DataChanged) &&
+ llvm::isa<llvm::omp::target::TaskGraphLiteralArgTy>(
+ G->Plan.getArg(U.Arg).get());
+ }))
+ ODBG(ODT_Interface) << "taskgraph " << G->GraphId
+ << " reparametrizes for a changed literal argument";
+ if ((!Updates.empty() || Resync) &&
+ Device.RTL->reparametrize_taskgraph(Device.RTLDeviceID, G, Updates,
+ Resync) != OFFLOAD_SUCCESS) {
+ Device.RTL->unpin_taskgraph(Device.RTLDeviceID, G);
+ return decline("the device could not adopt the current mapping state");
+ }
+ commitTaskGraphUpdates(*G);
+ Mappings.release();
+
+ // Past this point the backend may have issued part of the graph, so a
+ // failure cannot fall back to the host.
+ int32_t Rc = Device.RTL->replay_taskgraph(Device.RTLDeviceID, G);
+ if (Device.RTL->unpin_taskgraph(Device.RTLDeviceID, G) != OFFLOAD_SUCCESS)
+ Rc = OFFLOAD_FAIL;
+ if (Rc != OFFLOAD_SUCCESS)
FATAL_MESSAGE(G->DeviceId, "%s", "taskgraph replay failed");
+
+ // The graph has run, so its hold on the mappings can go.
+ releaseTaskGraphMappings(Device, *G);
return OFFLOAD_SUCCESS;
}
@@ -964,10 +1032,12 @@ EXTERN void __tgt_taskgraph_destroy(void *Graph) {
// DeviceId was resolved to a concrete device by finalize, which is also the
// only thing that can set PluginGraph.
auto DeviceOrErr = PM->getDevice(G->DeviceId);
- if (DeviceOrErr)
+ if (DeviceOrErr) {
DeviceOrErr->RTL->destroy_taskgraph(DeviceOrErr->RTLDeviceID, G);
- else
+ releaseTaskGraphPlan(*DeviceOrErr, *G);
+ } else {
consumeError(DeviceOrErr.takeError());
+ }
}
delete G;
}
diff --git a/offload/libomptarget/omptarget.cpp b/offload/libomptarget/omptarget.cpp
index a1555f1111b4b9..bb71028e974e91 100644
--- a/offload/libomptarget/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -665,6 +665,8 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
reinterpret_cast<uintptr_t>(TgtPtrBegin) - Delta);
ODBG(ODT_Mapping) << "Returning device pointer " << TgtPtrBase;
} else {
+ Device.getMappingInfo().warnIfClaimedByTaskGraph(
+ HstPtrBegin, 0, "use_device_ptr/use_device_addr");
// Lookup failed. So we have to decide what to do based on the
// requested fallback behavior.
//
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 590838efef374d..c450a97a57d6b2 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -1267,6 +1267,10 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
RecordReplayTy *getRecordReplay() { return RecordReplay; }
+ /// Whether this device can lower taskgraphs at all. libomptarget only builds
+ /// a graph's data plan, which allocates device memory, for a device that can.
+ virtual bool supportsTaskGraphs() const { return false; }
+
/// Taskgraph handoff hooks. \p Graph is a shared taskgraph tree
/// (Shared/TaskGraph.h) of target constructs only, which the plugin only
/// reads. libomptarget only offers a well-formed graph: every region has the
@@ -1294,6 +1298,44 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
return Plugin::success();
}
+ /// Take whatever hold the backend's copies need on the host memory of
+ /// \p Graph's plan ranges for one replay, such as locking the pages. Called
+ /// before every replay handed to the backend, after the plan is resolved and
+ /// before reparametrizeTaskGraph. Balanced by exactly one unpinTaskGraph,
+ /// after the replay or before a later decline. A range's host memory is
+ /// only known to be the program's between the two calls.
+ ///
+ /// A failure declines the replay to the host and must leave nothing held; no
+ /// unpinTaskGraph follows it.
+ virtual Error pinTaskGraph(const llvm::omp::target::TaskGraphTy *Graph) {
+ return Plugin::success();
+ }
+ /// Release what pinTaskGraph took, once nothing the replay issued can touch
+ /// the host ranges.
+ virtual Error unpinTaskGraph(const llvm::omp::target::TaskGraphTy *Graph) {
+ return Plugin::success();
+ }
+
+ /// Adopt the changes libomptarget has made to \p Graph's data plan: the
+ /// argument entries listed in \p Updates, or, with \p Resync, anything at
+ /// all, because a replay the backend never saw may have changed the plan.
+ /// Called before a replay handed to the backend, after pinTaskGraph,
+ /// whenever something changed. This is the last point at which the replay
+ /// can still decline to the host.
+ ///
+ /// A failure declines the replay, and the next replay to reach the backend
+ /// is a resync. An update only repoints addresses, refreshes bytes the plan
+ /// supplies and switches operations on or off; see TaskGraphArgUpdateTy.
+ /// The base implementation fails, since a backend that claimed a graph must
+ /// follow its plan.
+ virtual Error reparametrizeTaskGraph(
+ const llvm::omp::target::TaskGraphTy *Graph,
+ ArrayRef<llvm::omp::target::TaskGraphArgUpdateTy> Updates, bool Resync) {
+ return Plugin::error(error::ErrorCode::UNSUPPORTED,
+ "taskgraph reparametrization unsupported by this "
+ "plugin, which should not have claimed a graph");
+ }
+
/// Map to record kernel have been launchedl, for error reporting purposes.
ProtectedObj<KernelTraceInfoRecordTy> KernelLaunchTraces;
@@ -1600,6 +1642,7 @@ struct GenericPluginTy {
/// handled the request; libomp replays the graph on the host otherwise. A
/// successful finalize returns the plugin's form of the graph in
/// \p PluginGraph.
+ int32_t supports_taskgraphs(int32_t DeviceId);
int32_t finalize_taskgraph(int32_t DeviceId,
const llvm::omp::target::TaskGraphTy *Graph,
void **PluginGraph);
@@ -1607,6 +1650,13 @@ struct GenericPluginTy {
const llvm::omp::target::TaskGraphTy *Graph);
int32_t destroy_taskgraph(int32_t DeviceId,
const llvm::omp::target::TaskGraphTy *Graph);
+ int32_t reparametrize_taskgraph(
+ int32_t DeviceId, const llvm::omp::target::TaskGraphTy *Graph,
+ ArrayRef<llvm::omp::target::TaskGraphArgUpdateTy> Updates, bool Resync);
+ int32_t pin_taskgraph(int32_t DeviceId,
+ const llvm::omp::target::TaskGraphTy *Graph);
+ int32_t unpin_taskgraph(int32_t DeviceId,
+ const llvm::omp::target::TaskGraphTy *Graph);
/// Returns if the plugin can support automatic copy.
int32_t use_auto_zero_copy(int32_t DeviceId);
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 73dee99ce39e87..38694c8552718d 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -1231,6 +1231,10 @@ int32_t GenericPluginTy::launch_kernel(int32_t DeviceId, void *TgtEntryPtr,
return OFFLOAD_SUCCESS;
}
+int32_t GenericPluginTy::supports_taskgraphs(int32_t DeviceId) {
+ return getDevice(DeviceId).supportsTaskGraphs();
+}
+
int32_t
GenericPluginTy::finalize_taskgraph(int32_t DeviceId,
const llvm::omp::target::TaskGraphTy *Graph,
@@ -1263,6 +1267,42 @@ GenericPluginTy::replay_taskgraph(int32_t DeviceId,
return OFFLOAD_SUCCESS;
}
+int32_t GenericPluginTy::reparametrize_taskgraph(
+ int32_t DeviceId, const llvm::omp::target::TaskGraphTy *Graph,
+ ArrayRef<llvm::omp::target::TaskGraphArgUpdateTy> Updates, bool Resync) {
+ auto Err = getDevice(DeviceId).reparametrizeTaskGraph(Graph, Updates, Resync);
+ if (Err) {
+ REPORT() << "Failure to reparametrize taskgraph on device " << DeviceId
+ << ": " << toString(std::move(Err));
+ return OFFLOAD_FAIL;
+ }
+ return OFFLOAD_SUCCESS;
+}
+
+int32_t
+GenericPluginTy::pin_taskgraph(int32_t DeviceId,
+ const llvm::omp::target::TaskGraphTy *Graph) {
+ auto Err = getDevice(DeviceId).pinTaskGraph(Graph);
+ if (Err) {
+ REPORT() << "Failure to pin taskgraph host ranges on device " << DeviceId
+ << ": " << toString(std::move(Err));
+ return OFFLOAD_FAIL;
+ }
+ return OFFLOAD_SUCCESS;
+}
+
+int32_t
+GenericPluginTy::unpin_taskgraph(int32_t DeviceId,
+ const llvm::omp::target::TaskGraphTy *Graph) {
+ auto Err = getDevice(DeviceId).unpinTaskGraph(Graph);
+ if (Err) {
+ REPORT() << "Failure to unpin taskgraph host ranges on device " << DeviceId
+ << ": " << toString(std::move(Err));
+ return OFFLOAD_FAIL;
+ }
+ return OFFLOAD_SUCCESS;
+}
+
int32_t GenericPluginTy::destroy_taskgraph(
int32_t DeviceId, const llvm::omp::target::TaskGraphTy *Graph) {
auto Err = getDevice(DeviceId).destroyTaskGraph(Graph);
diff --git a/offload/test/offloading/taskgraph_claim_attach_demotion.c b/offload/test/offloading/taskgraph_claim_attach_demotion.c
new file mode 100644
index 00000000000000..294b77d9f4dc18
--- /dev/null
+++ b/offload/test/offloading/taskgraph_claim_attach_demotion.c
@@ -0,0 +1,83 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=NOGRAPH
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=GRAPH
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A pointee a taskgraph would keep to itself, attached to a pointer the
+// program has mapped. The device copy of G.P outlives every replay, so were
+// the pointee in storage of the graph's own it would be left pointing at
+// memory freed when the replay ended; the graph maps that pointee through the
+// mapping table instead, as it would a region found mapped. L.P's pointee has
+// no such pointer -- L is mapped by the graph alone -- so it stays the graph's.
+//
+// Either way the graph holds neither once the subgraph has run, so the lookup
+// APIs, asked from a host task ordered after the construct, find L.P's pointee
+// unmapped as they would outside a graph, and nothing warns that the graph has
+// it. Only the answer about L.P's pointee is compared: G.P's is mapped by the
+// `target enter data` of G for as long as G is.
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 256
+#define ITERS 3
+
+struct S {
+ int *P;
+ int Tag;
+};
+
+static struct S G;
+static int *DataG, *DataL;
+static int Dev, Hits;
+
+static void step(void) {
+ struct S L;
+ L.P = DataL;
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : G.P[0 : N]) map(tofrom : L.P[0 : N]) \
+ depend(out : G)
+ for (int I = 0; I < N; ++I) {
+ G.P[I] += 1;
+ L.P[I] += 2;
+ }
+#pragma omp task depend(in : G)
+ {
+ (void)omp_get_mapped_ptr(DataG, Dev);
+ Hits += omp_target_is_present(DataL, Dev);
+ }
+ }
+}
+
+int main(void) {
+ DataG = (int *)calloc(N, sizeof(int));
+ DataL = (int *)calloc(N, sizeof(int));
+ G.P = DataG;
+ Dev = omp_get_default_device();
+
+#pragma omp target enter data map(to : G)
+ for (int It = 0; It < ITERS; ++It)
+ step();
+#pragma omp target exit data map(release : G)
+
+ int Errors = 0;
+ for (int I = 0; I < N; ++I)
+ Errors += DataG[I] != ITERS || DataL[I] != 2 * ITERS;
+
+ // NOGRAPH-NOT: asked about
+ // GRAPH-NOT: asked about
+
+ // NOGRAPH: hits 0 errors 0
+ // GRAPH: hits 0 errors 0
+ printf("hits %d errors %d\n", Hits, Errors);
+ free(DataL);
+ free(DataG);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_claim_enter_data.c b/offload/test/offloading/taskgraph_claim_enter_data.c
new file mode 100644
index 00000000000000..465b13620f5fba
--- /dev/null
+++ b/offload/test/offloading/taskgraph_claim_enter_data.c
@@ -0,0 +1,59 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=NOGRAPH
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=GRAPH
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// The program mapping, between replays, what a taskgraph keeps to itself. A
+// is unmapped when the graph is finalized, so the graph gives it storage of its
+// own on each replay; then a `target enter data` maps it between replays. The
+// graph holds nothing between replays, so the mapping is not one it can object
+// to, and the replay that finds it borrows it, just as the graph's `target`
+// region would share it outside a graph: the graph's own copies of A are off
+// for that replay, the mapping's `to` and `from` standing in for them. The
+// replay after the `target exit data` is back on storage of the graph's own.
+//
+// None of this declines a replay or warns: B, which the graph keeps to itself
+// throughout, is not affected either.
+
+#include <stdio.h>
+
+#define N 256
+#define ITERS 5
+
+static int A[N], B[N];
+
+int main(void) {
+ for (int It = 0; It < ITERS; ++It) {
+ if (It == 2) {
+#pragma omp target enter data map(to : A[0 : N])
+ }
+ if (It == 3) {
+#pragma omp target exit data map(from : A[0 : N])
+ }
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A[0 : N], B[0 : N])
+ for (int I = 0; I < N; ++I) {
+ A[I] += 1;
+ B[I] += 2;
+ }
+ }
+ }
+
+ int Errors = 0;
+ for (int I = 0; I < N; ++I)
+ Errors += A[I] != ITERS || B[I] != 2 * ITERS;
+
+ // NOGRAPH-NOT: taskgraph
+ // GRAPH-NOT: taskgraph
+
+ // NOGRAPH: taskgraph claim enter data errors: 0
+ // GRAPH: taskgraph claim enter data errors: 0
+ printf("taskgraph claim enter data errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_claim_finalize_partial.c b/offload/test/offloading/taskgraph_claim_finalize_partial.c
new file mode 100644
index 00000000000000..e836215fe9230f
--- /dev/null
+++ b/offload/test/offloading/taskgraph_claim_finalize_partial.c
@@ -0,0 +1,77 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=NOGRAPH
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=GRAPH
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A taskgraph region that is only partly mapped outside the graph when the
+// graph is finalized. Whether the graph keeps a region to itself or borrows
+// the program's mapping of it is decided then, once, since the two lower to
+// different graphs -- and a region mapped only in part is neither: there is no
+// mapping covering it to borrow, and storage of the graph's own would be a
+// second device copy of the part that is mapped. So the graph is declined
+// when it is finalized, and every replay runs on the host.
+//
+// The partial mapping is made by the graph's last host task and undone by its
+// first, so that it exists when the graph is finalized and between replays but
+// never while the graph's `target` region maps [N / 4, 3 * N / 4), which it
+// would otherwise refuse as an extension of mapped data.
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 256
+#define ITERS 3
+
+static int A[N];
+static int Dev, Associated, Failures;
+static void *DevMem;
+
+int main(void) {
+ Dev = omp_get_default_device();
+ DevMem = omp_target_alloc(N / 2 * sizeof(int), Dev);
+
+ for (int It = 0; It < ITERS; ++It) {
+#pragma omp taskgraph
+ {
+#pragma omp task depend(out : A[0])
+ if (Associated) {
+ Failures += omp_target_disassociate_ptr(A, Dev) != 0;
+ Associated = 0;
+ }
+#pragma omp target map(tofrom : A[N / 4 : N / 2]) depend(inout : A[0])
+ for (int I = N / 4; I < 3 * N / 4; ++I)
+ A[I] += 1;
+#pragma omp task depend(in : A[0])
+ {
+ Failures += omp_target_associate_ptr(A, DevMem, N / 2 * sizeof(int), 0,
+ Dev) != 0;
+ Associated = 1;
+ }
+ }
+ }
+
+ if (Associated)
+ Failures += omp_target_disassociate_ptr(A, Dev) != 0;
+ omp_target_free(DevMem, Dev);
+
+ int Errors = Failures;
+ for (int I = 0; I < N; ++I)
+ Errors += A[I] != (I >= N / 4 && I < 3 * N / 4 ? ITERS : 0);
+
+ // NOGRAPH-NOT: taskgraph
+ // GRAPH: taskgraph {{[0-9]+}} replays on the host: [{{.*}}) is only partly mapped outside the graph
+ // GRAPH-NOT: taskgraph
+
+ // NOGRAPH: taskgraph claim finalize partial errors: 0
+ // GRAPH: taskgraph claim finalize partial errors: 0
+ printf("taskgraph claim finalize partial errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_claim_lookup.c b/offload/test/offloading/taskgraph_claim_lookup.c
new file mode 100644
index 00000000000000..9cd9e21727067f
--- /dev/null
+++ b/offload/test/offloading/taskgraph_claim_lookup.c
@@ -0,0 +1,58 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=NOGRAPH
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=GRAPH
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A region a taskgraph maps, balances and finds unmapped when it is finalized
+// is one the graph keeps to itself: each replay gives it device storage of its
+// own, and there is no mapping-table entry for the program to find. So asking
+// for it gets the answer an unmapped region gets -- which is also what the
+// same program gets outside a graph, where the region is mapped only while its
+// `target` region runs. Only a question asked while the subgraph holding the
+// region runs would come with a warning that the graph has it. Once the
+// subgraph has run the graph holds nothing, and the region is unmapped just as
+// it would be outside a graph -- which is where both questions here are asked:
+// B's between replays, and A's from a host task ordered after the construct.
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 256
+#define ITERS 3
+
+static int A[N], B[N];
+static int Dev, DuringHits, BetweenHits;
+
+int main(void) {
+ Dev = omp_get_default_device();
+ for (int It = 0; It < ITERS; ++It) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A[0 : N], B[0 : N]) depend(inout : A[0])
+ for (int I = 0; I < N; ++I) {
+ A[I] += 1;
+ B[I] += 2;
+ }
+#pragma omp task depend(in : A[0])
+ DuringHits += omp_get_mapped_ptr(A, Dev) != NULL;
+ }
+ BetweenHits += omp_target_is_present(B, Dev);
+ }
+
+ int Errors = 0;
+ for (int I = 0; I < N; ++I)
+ Errors += A[I] != ITERS || B[I] != 2 * ITERS;
+
+ // NOGRAPH-NOT: asked about
+ // GRAPH-NOT: asked about
+
+ // NOGRAPH: during 0 between 0 errors 0
+ // GRAPH: during 0 between 0 errors 0
+ printf("during %d between %d errors %d\n", DuringHits, BetweenHits, Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_claim_overlapping_graphs.c b/offload/test/offloading/taskgraph_claim_overlapping_graphs.c
new file mode 100644
index 00000000000000..caee41f1402efc
--- /dev/null
+++ b/offload/test/offloading/taskgraph_claim_overlapping_graphs.c
@@ -0,0 +1,64 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=NOGRAPH
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=GRAPH
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Two taskgraphs keeping overlapping memory to themselves. Neither is mapped
+// outside its own replays, so each replay has its own storage and they take
+// turns without either seeing the other's -- exactly as two `target` regions
+// mapping the overlapping sections one after the other do outside a graph.
+//
+// A graph has its memory only while it replays, and each replay here is over
+// before the next graph starts, so nothing the program does between them is
+// warned about: not the second graph's recording encounter, which maps its
+// section like any host code, and not the lookups once both are done.
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 256
+#define ITERS 3
+
+static int A[N + N / 2];
+
+int main(void) {
+ int Dev = omp_get_default_device();
+ for (int It = 0; It < ITERS; ++It) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A[0 : N])
+ for (int I = 0; I < N; ++I)
+ A[I] += 1;
+ }
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A[N / 2 : N])
+ for (int I = N / 2; I < N + N / 2; ++I)
+ A[I] += 2;
+ }
+ }
+
+ int Hits = 0;
+ Hits += omp_target_is_present(&A[0], Dev);
+ Hits += omp_target_is_present(&A[1], Dev);
+ Hits += omp_target_is_present(&A[N / 2], Dev);
+ Hits += omp_target_is_present(&A[N], Dev);
+
+ int Errors = 0;
+ for (int I = 0; I < N + N / 2; ++I)
+ Errors += A[I] != (I < N ? ITERS : 0) + (I >= N / 2 ? 2 * ITERS : 0);
+
+ // NOGRAPH-NOT: taskgraph
+ // GRAPH-NOT: taskgraph
+ // GRAPH-NOT: asked about
+
+ // NOGRAPH: hits 0 errors 0
+ // GRAPH: hits 0 errors 0
+ printf("hits %d errors %d\n", Hits, Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_claim_relocated.c b/offload/test/offloading/taskgraph_claim_relocated.c
new file mode 100644
index 00000000000000..c8940c571e1ab5
--- /dev/null
+++ b/offload/test/offloading/taskgraph_claim_relocated.c
@@ -0,0 +1,68 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=NOGRAPH
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=GRAPH
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// What a taskgraph keeps to itself is claimed only while the subgraph holding
+// it runs. The graph maps a stack array, replayed from frames at different
+// depths, so relocation moves the graph's range with the array -- and once the
+// target construct has run, the claim has to be gone from whichever frame the
+// replay used. A host task ordered after the construct runs once the subgraph
+// is over, so asking about the array there finds it unmapped and unclaimed, as
+// it would be outside a graph, and nothing warns.
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 256
+
+static int Dev, Hits, Errors;
+
+static void work(void) {
+ int Local[N];
+ for (int I = 0; I < N; ++I)
+ Local[I] = I;
+
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : Local[0 : N]) depend(inout : Local[0])
+ for (int I = 0; I < N; ++I)
+ Local[I] += 1;
+#pragma omp task shared(Local) depend(in : Local[0])
+ Hits += omp_target_is_present(Local, Dev);
+ }
+
+ for (int I = 0; I < N; ++I)
+ Errors += Local[I] != I + 1;
+}
+
+static void atDepth(int Depth) {
+ volatile char Pad[4096];
+ Pad[0] = (char)Depth;
+ if (Depth)
+ atDepth(Depth - 1);
+ else
+ work();
+ Errors += Pad[0] != (char)Depth;
+}
+
+int main(void) {
+ Dev = omp_get_default_device();
+ atDepth(0);
+ atDepth(2);
+ atDepth(2);
+ atDepth(5);
+
+ // NOGRAPH-NOT: asked about
+ // GRAPH-NOT: asked about
+
+ // NOGRAPH: hits 0 errors 0
+ // GRAPH: hits 0 errors 0
+ printf("hits %d errors %d\n", Hits, Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_decline_info.c b/offload/test/offloading/taskgraph_decline_info.c
new file mode 100644
index 00000000000000..067581eaff3cdf
--- /dev/null
+++ b/offload/test/offloading/taskgraph_decline_info.c
@@ -0,0 +1,81 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_INFO=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A graph mapping something the data plan cannot express is replayed by libomp
+// on the host instead, which is correct but gives up the device graph. Each
+// such map must say so under LIBOMPTARGET_INFO, naming what was wrong with it,
+// rather than falling back without a trace; and the results must still be
+// right.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 64
+
+typedef struct {
+ int Len;
+ int *Data;
+} VecTy;
+
+#pragma omp declare mapper(VecTy V) map(V, V.Data[0 : V.Len])
+
+int main() {
+ const int Iters = 3;
+ int errors = 0;
+
+ VecTy V;
+ V.Len = N;
+ V.Data = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ V.Data[i] = i;
+
+ // CHECK: taskgraph {{[0-9]+}} replays on the host: argument {{[0-9]+}} of leaf 0 ({{.*}}) uses a user-defined mapper
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : V)
+ for (int i = 0; i < V.Len; ++i)
+ V.Data[i] += 1;
+ }
+ }
+ for (int i = 0; i < N; ++i)
+ if (V.Data[i] != i + Iters)
+ ++errors;
+
+ // A strided update copies every other element. The odd ones keep whatever
+ // the device last had, which is what the region wrote before the loop.
+ int A[N];
+ for (int i = 0; i < N; ++i)
+ A[i] = 0;
+#pragma omp target data map(tofrom : A[0 : N])
+ {
+#pragma omp target
+ for (int i = 0; i < N; ++i)
+ A[i] = 100;
+ for (int it = 0; it < Iters; ++it) {
+ for (int i = 0; i < N; ++i)
+ A[i] = it;
+ // CHECK: taskgraph {{[0-9]+}} replays on the host: argument {{[0-9]+}} of leaf 0 ({{.*}}) is a non-contiguous transfer
+#pragma omp taskgraph
+ {
+#pragma omp target update to(A[0 : N / 2 : 2])
+ }
+ }
+ }
+ for (int i = 0; i < N; ++i)
+ if (A[i] != (i % 2 ? 100 : Iters - 1))
+ ++errors;
+
+ // CHECK: taskgraph decline info errors: 0
+ printf("taskgraph decline info errors: %d\n", errors);
+ free(V.Data);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_enter_data_struct_member.c b/offload/test/offloading/taskgraph_enter_data_struct_member.c
new file mode 100644
index 00000000000000..cb6a878df857be
--- /dev/null
+++ b/offload/test/offloading/taskgraph_enter_data_struct_member.c
@@ -0,0 +1,56 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A graph that maps struct members with `target enter data` and unmaps them
+// with `target exit data`. Such a map is a struct entry followed by its
+// members, and only the struct entry changes the reference count: a member
+// copies on whichever change its struct's entry owns. So every replay copies
+// the members in on the enter and out on the exit, and the device never shows
+// a replay what the previous one left behind.
+//
+// The host changes the data between replays, so a missing copy-in shows as the
+// previous replay's values. `e` is released rather than copied back, so it
+// must come back unchanged.
+
+#include <stdio.h>
+
+#define N 4
+#define ITERS 4
+
+struct S {
+ int Pad;
+ int D[N];
+ int E[N];
+};
+
+int main(void) {
+ struct S s = {0};
+ int Errors = 0;
+
+ for (int It = 0; It < ITERS; ++It) {
+ for (int I = 0; I < N; ++I)
+ s.D[I] = s.E[I] = 100 * It + I;
+#pragma omp taskgraph
+ {
+#pragma omp target enter data map(to : s.D[0 : N], s.E[0 : N])
+#pragma omp target map(present, alloc : s.D[0 : N], s.E[0 : N])
+ for (int I = 0; I < N; ++I) {
+ s.D[I] += 1;
+ s.E[I] += 2;
+ }
+#pragma omp target exit data map(from : s.D[0 : N]) map(release : s.E[0 : N])
+ }
+ for (int I = 0; I < N; ++I)
+ Errors += s.D[I] != 100 * It + I + 1 || s.E[I] != 100 * It + I;
+ }
+
+ // CHECK: taskgraph enter data struct member errors: 0
+ printf("taskgraph enter data struct member errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_enter_data_unbalanced.c b/offload/test/offloading/taskgraph_enter_data_unbalanced.c
new file mode 100644
index 00000000000000..8d87f8752b441d
--- /dev/null
+++ b/offload/test/offloading/taskgraph_enter_data_unbalanced.c
@@ -0,0 +1,73 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A `target enter data` with no matching exit inside the region maps something
+// that outlives the replay. That makes the graph's maps a change to the
+// program's mapping state rather than a private arrangement, so a replay has to
+// do the reference-count bookkeeping for real: the mapping has to be there when
+// the region ends, and an ordinary `target exit data` outside the graph has to
+// be able to remove it.
+//
+// It is also the case that distinguishes a replay which really performs the
+// enter-data from one that merely arranged its transfers. The `present` map on
+// the kernel beside it says the mapping the graph makes is visible to the rest
+// of the graph, and the value check says the `to` copy happened on this replay
+// rather than once at record time -- the reference count reaching zero at the
+// end of each iteration is what puts the copy back in play.
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 256
+#define ITERS 3
+
+int main(void) {
+ int Dev = omp_get_default_device();
+ int *A = (int *)malloc(N * sizeof(int));
+
+ int Mapped = 0, Removed = 0, Errors = 0;
+
+ for (int It = 0; It < ITERS; ++It) {
+ for (int I = 0; I < N; ++I)
+ A[I] = 100 * (It + 1) + I;
+
+#pragma omp taskgraph
+ {
+#pragma omp target enter data map(to : A[0 : N])
+#pragma omp target map(present, alloc : A[0 : N])
+ for (int I = 0; I < N; ++I)
+ A[I] += 1;
+ }
+
+ if (omp_get_mapped_ptr(A, Dev))
+ ++Mapped;
+
+ // Balanced from out here, against a mapping made in there.
+#pragma omp target exit data map(from : A[0 : N])
+
+ if (!omp_get_mapped_ptr(A, Dev))
+ ++Removed;
+
+ for (int I = 0; I < N; ++I)
+ if (A[I] != 100 * (It + 1) + I + 1)
+ ++Errors;
+ }
+
+ // CHECK: still mapped when the region ended: 3 of 3
+ printf("still mapped when the region ended: %d of %d\n", Mapped, ITERS);
+ // CHECK: removed by the exit data outside: 3 of 3
+ printf("removed by the exit data outside: %d of %d\n", Removed, ITERS);
+ // CHECK: taskgraph unbalanced enter data errors: 0
+ printf("taskgraph unbalanced enter data errors: %d\n", Errors);
+
+ free(A);
+ return Errors != 0 || Mapped != ITERS || Removed != ITERS;
+}
diff --git a/offload/test/offloading/taskgraph_exit_data_borrowed.c b/offload/test/offloading/taskgraph_exit_data_borrowed.c
new file mode 100644
index 00000000000000..5d16e19db98c6e
--- /dev/null
+++ b/offload/test/offloading/taskgraph_exit_data_borrowed.c
@@ -0,0 +1,82 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env KMP_TASKGRAPH_TRACE=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefixes=CHECK,TRACE \
+// RUN: --implicit-check-not='replays on the host'
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A `target exit data` in a graph that drops a reference the graph did not
+// take. Outside a graph the decrement that removes a mapping is the one that
+// copies it out, so whether this one copies depends on the mapping the replay
+// finds: A's only reference is the one the graph drops, so the graph removes
+// A's mapping and copies A out; B has a second reference, so B's mapping stays
+// and nothing is copied until the program drops that one too.
+//
+// This is the shape the exit-data of a `target data` takes once the body is
+// run between subgraphs, the enter-data having mapped the data in an earlier
+// one -- so it is common, not contrived, and getting it wrong silently loses
+// everything the body did on the device.
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 256
+#define ITERS 4
+
+static int A[N], B[N];
+
+int main(void) {
+ int Dev = omp_get_default_device();
+ int BadLast = 0, BadNotLast = 0, LeftMapped = 0, Unmapped = 0;
+
+ for (int It = 0; It < ITERS; ++It) {
+ for (int I = 0; I < N; ++I)
+ A[I] = B[I] = I;
+
+#pragma omp target enter data map(to : A[0 : N])
+#pragma omp target enter data map(to : B[0 : N])
+#pragma omp target enter data map(to : B[0 : N])
+#pragma omp target map(present, alloc : A[0 : N], B[0 : N])
+ for (int I = 0; I < N; ++I) {
+ A[I] += It + 1;
+ B[I] += It + 1;
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp target exit data map(from : A[0 : N], B[0 : N])
+ }
+
+ for (int I = 0; I < N; ++I) {
+ if (A[I] != I + It + 1)
+ ++BadLast;
+ if (B[I] != I)
+ ++BadNotLast;
+ }
+ LeftMapped += omp_target_is_present(A, Dev);
+ Unmapped += !omp_target_is_present(B, Dev);
+
+#pragma omp target exit data map(from : B[0 : N])
+ for (int I = 0; I < N; ++I)
+ if (B[I] != I + It + 1)
+ ++BadNotLast;
+ }
+
+ // TRACE: Transmitted taskgraph {{.*}} subgraph
+
+ // CHECK: last reference not copied out: 0
+ printf("last reference not copied out: %d\n", BadLast);
+ // CHECK: other reference copied out early: 0
+ printf("other reference copied out early: %d\n", BadNotLast);
+ // CHECK: left mapped: 0, unmapped early: 0
+ printf("left mapped: %d, unmapped early: %d\n", LeftMapped, Unmapped);
+ return BadLast + BadNotLast + LeftMapped + Unmapped != 0;
+}
diff --git a/offload/test/offloading/taskgraph_firstprivate_inner.c b/offload/test/offloading/taskgraph_firstprivate_inner.c
new file mode 100644
index 00000000000000..886b1f09d7194d
--- /dev/null
+++ b/offload/test/offloading/taskgraph_firstprivate_inner.c
@@ -0,0 +1,91 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=CHECK
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefixes=CHECK,GRAPH
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A replay starts from the taskgraph construct's enclosing data environment,
+// and a variable declared inside the region is not part of it: the code that
+// declared and initialised it does not run again. So a firstprivate of such a
+// variable takes its value from the saved data environment (OpenMP 6.0
+// [14.3]), as though the clause had said `saved`, rather than from wherever its
+// storage happened to be when the graph was recorded.
+//
+// Each replay runs from a different stack depth, and one of them is declined
+// to the host, so neither the recording frame nor the device's copy can stand
+// in for the saved value by accident.
+
+#include <stdio.h>
+#include <string.h>
+
+#define M 16
+
+struct Big {
+ int V[M];
+};
+
+struct Holder {
+ int *P;
+};
+static int Out;
+static struct Holder H = {&Out};
+
+static void run(int N) {
+#pragma omp taskgraph
+ {
+ struct Big C;
+ for (int I = 0; I < M; ++I)
+ C.V[I] = N;
+#pragma omp target firstprivate(C) map(tofrom : H.P[0 : 1])
+ H.P[0] = C.V[M - 1];
+ }
+}
+
+// Replays from Depth frames further down, over stack that has been scribbled
+// on first.
+static void deeper(int Depth, int N) {
+ volatile char Scribble[512];
+ memset((char *)Scribble, 0x5a, sizeof(Scribble));
+ if (Depth)
+ deeper(Depth - 1, N);
+ else
+ run(N);
+}
+
+int main(void) {
+ const int Declined = 2;
+ int Errors = 0;
+ for (int It = 0; It < 5; ++It) {
+ // Mapping H from outside the graph declines this one replay: the graph
+ // would borrow that mapping of the pointer, and attaching it to the
+ // pointee the graph keeps to itself would leave it pointing at memory
+ // freed when the replay ends.
+ if (It == Declined) {
+#pragma omp target enter data map(to : H)
+ }
+ Out = -1;
+ deeper(It, 100 + It);
+ if (It == Declined) {
+#pragma omp target exit data map(release : H)
+ }
+ if (Out != 100) {
+ printf("replay %d: C %d (want 100)\n", It, Out);
+ ++Errors;
+ }
+ }
+
+ // GRAPH: replays on the host this time
+ // GRAPH-NOT: replays on the host this time
+ // CHECK-NOT: replay {{[0-9]+}}:
+ // CHECK: taskgraph firstprivate inner errors: 0
+ printf("taskgraph firstprivate inner errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_firstprivate_replay_start.c b/offload/test/offloading/taskgraph_firstprivate_replay_start.c
new file mode 100644
index 00000000000000..68ad567bbd61ef
--- /dev/null
+++ b/offload/test/offloading/taskgraph_firstprivate_replay_start.c
@@ -0,0 +1,57 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --implicit-check-not='replays on the host'
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A replay re-encounters every recorded construct before any of its tasks
+// runs (OpenMP 6.0 [14.3]), so a target region's by-value firstprivate takes
+// the value its variable has when the replay starts, not the value a task it
+// depends on has written by the time it is launched. Both ways of issuing
+// the region have to agree on this: in the first run the plugin claims no
+// graph and libomp issues the region itself, and in the second the region is
+// replayed as a device subgraph, after the writer, a host task libomp runs.
+//
+// Only the replays are checked: when the graph is recorded, the writer may
+// run before the target region is encountered, and then the region sees the
+// write.
+
+#include <stdio.h>
+
+#pragma omp begin declare target
+int Out;
+#pragma omp end declare target
+
+static void run(int N) {
+#pragma omp taskgraph
+ {
+#pragma omp task shared(N) depend(out : N)
+ N = -1;
+#pragma omp target firstprivate(N) depend(in : N)
+ Out = N;
+ }
+}
+
+int main(void) {
+ int Errors = 0;
+ for (int It = 1; It <= 4; ++It) {
+ Out = -2;
+#pragma omp target update to(Out)
+ run(It);
+#pragma omp target update from(Out)
+ if (It > 1 && Out != It) {
+ printf("replay %d: N %d (want %d)\n", It, Out, It);
+ ++Errors;
+ }
+ }
+
+ // CHECK-NOT: replay {{[0-9]+}}:
+ // CHECK: taskgraph firstprivate replay start errors: 0
+ printf("taskgraph firstprivate replay start errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_firstprivate_restage.c b/offload/test/offloading/taskgraph_firstprivate_restage.c
new file mode 100644
index 00000000000000..de6cfaa2acb102
--- /dev/null
+++ b/offload/test/offloading/taskgraph_firstprivate_restage.c
@@ -0,0 +1,89 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=CHECK
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefixes=CHECK,GRAPH
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A graph copies a firstprivate aggregate into its private storage only when
+// the value it is initialised from has changed since the device last saw it,
+// since nothing on the device writes that storage. So the value has to be
+// tracked against what the device holds, not against what was last staged.
+//
+// B is an ordinary firstprivate of a variable the taskgraph construct's
+// enclosing data environment has, so each replay sees it as it stands then: it
+// repeats a value, returns to an earlier one, and changes on a replay that is
+// declined to the host, then keeps that value on the replay after, which the
+// device has not seen yet. S is `saved`, so every replay sees the value it had
+// when the graph was recorded, however the variable has changed since.
+
+#include <stdio.h>
+
+#define M 16
+
+struct Big {
+ int V[M];
+};
+
+struct Holder {
+ int *P;
+};
+static int Out[2];
+static struct Holder H = {Out};
+
+static void run(struct Big B, struct Big S) {
+#pragma omp taskgraph
+ {
+#pragma omp target firstprivate(B) firstprivate(saved : S) \
+ map(tofrom : H.P[0 : 2])
+ {
+ H.P[0] = B.V[M - 1];
+ H.P[1] = S.V[M - 1];
+ }
+ }
+}
+
+static struct Big fill(int X) {
+ struct Big R;
+ for (int I = 0; I < M; ++I)
+ R.V[I] = X;
+ return R;
+}
+
+int main(void) {
+ const int Values[] = {1, 2, 2, 1, 3, 3, 3, 4};
+ const int Declined = 4;
+ int Errors = 0;
+ for (int It = 0; It < (int)(sizeof(Values) / sizeof(Values[0])); ++It) {
+ // Mapping H from outside the graph declines this one replay: the graph
+ // would borrow that mapping of the pointer, and attaching it to the
+ // pointee the graph keeps to itself would leave it pointing at memory
+ // freed when the replay ends.
+ if (It == Declined) {
+#pragma omp target enter data map(to : H)
+ }
+ run(fill(Values[It]), fill(100 + It));
+ if (It == Declined) {
+#pragma omp target exit data map(release : H)
+ }
+ if (Out[0] != Values[It] || Out[1] != 100) {
+ printf("replay %d: B %d (want %d), S %d (want 100)\n", It, Out[0],
+ Values[It], Out[1]);
+ ++Errors;
+ }
+ }
+
+ // GRAPH: replays on the host this time
+ // GRAPH-NOT: replays on the host this time
+ // CHECK-NOT: replay {{[0-9]+}}:
+ // CHECK: taskgraph firstprivate restage errors: 0
+ printf("taskgraph firstprivate restage errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_firstprivate_scalar.c b/offload/test/offloading/taskgraph_firstprivate_scalar.c
new file mode 100644
index 00000000000000..4e61f01fa76b35
--- /dev/null
+++ b/offload/test/offloading/taskgraph_firstprivate_scalar.c
@@ -0,0 +1,102 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=CHECK
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefixes=CHECK,GRAPH
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A firstprivate scalar is passed to the kernel by value. When its variable is
+// in the taskgraph construct's enclosing data environment, each replay takes
+// the variable's value at that replay (OpenMP 6.0 [14.3]); a `saved` one keeps
+// the value it had when the graph was recorded.
+//
+// The replays run from different stack depths, and one of them is declined to
+// the host. The replay after the decline passes the same value as the
+// declined one: the declined replay has already brought the recorded arguments
+// up to date, so the value is right there only if the device remembers that it
+// has not taken the change in yet. The last replay repeats a value too, with
+// no decline in between.
+
+#include <stdio.h>
+#include <string.h>
+
+// The region using N maps nothing, writing its results to a declare target
+// variable instead, and the decline is set off through Trigger, which a
+// different region maps. So nothing about the declined replay touches the
+// region using N: a replay that re-reports every mapping leaves it alone, and
+// its kernel arguments are rewritten because N changed or not at all.
+#pragma omp begin declare target
+int Out[3];
+#pragma omp end declare target
+struct Holder {
+ int *P;
+};
+static int Count;
+static struct Holder Trigger = {&Count};
+
+static void run(int N, int S, double D) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : Trigger.P[0 : 1])
+ ++Trigger.P[0];
+#pragma omp target firstprivate(N, D) firstprivate(saved : S)
+ {
+ Out[0] = N;
+ Out[1] = S;
+ Out[2] = (int)D;
+ }
+ }
+}
+
+// Replays from Depth frames further down, over stack that has been scribbled
+// on first.
+static void deeper(int Depth, int N, int S, double D) {
+ volatile char Scribble[512];
+ memset((char *)Scribble, 0x5a, sizeof(Scribble));
+ if (Depth)
+ deeper(Depth - 1, N, S, D);
+ else
+ run(N, S, D);
+}
+
+int main(void) {
+ const int Declined = 2;
+ const int N[] = {1, 2, 3, 3, 4, 4};
+ int Errors = 0;
+ for (int It = 0; It < 6; ++It) {
+ // Mapping Trigger from outside the graph declines this one replay: the
+ // graph would borrow that mapping of the pointer, and attaching it to the
+ // pointee the graph keeps to itself would leave it pointing at memory
+ // freed when the replay ends.
+ if (It == Declined) {
+#pragma omp target enter data map(to : Trigger)
+ }
+ Out[0] = Out[1] = Out[2] = -1;
+#pragma omp target update to(Out)
+ deeper(It, N[It], 50 + It, 10.0 * N[It]);
+#pragma omp target update from(Out)
+ if (It == Declined) {
+#pragma omp target exit data map(release : Trigger)
+ }
+ if (Out[0] != N[It] || Out[1] != 50 || Out[2] != 10 * N[It]) {
+ printf("replay %d: N %d (want %d) S %d (want 50) D %d (want %d)\n", It,
+ Out[0], N[It], Out[1], Out[2], 10 * N[It]);
+ ++Errors;
+ }
+ }
+ Errors += Count != 6;
+
+ // GRAPH: replays on the host this time
+ // GRAPH-NOT: replays on the host this time
+ // CHECK-NOT: replay {{[0-9]+}}:
+ // CHECK: taskgraph firstprivate scalar errors: 0
+ printf("taskgraph firstprivate scalar errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_firstprivate_scalar_reparam.c b/offload/test/offloading/taskgraph_firstprivate_scalar_reparam.c
new file mode 100644
index 00000000000000..5fdc915f6b7ca7
--- /dev/null
+++ b/offload/test/offloading/taskgraph_firstprivate_scalar_reparam.c
@@ -0,0 +1,50 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_DEBUG=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// REQUIRES: libomptarget-debug
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// Refreshing a by-value firstprivate costs a kernel-argument rewrite only when
+// the value actually changed: a replay passing the value the previous one did
+// takes the graph as it is.
+
+#include <stdio.h>
+
+static int Out;
+
+static void run(int N) {
+#pragma omp taskgraph
+ {
+#pragma omp target firstprivate(N) map(tofrom : Out)
+ Out = N;
+ }
+}
+
+int main(void) {
+ const int N[] = {1, 1, 2, 2};
+ int Errors = 0;
+ for (int It = 0; It < 4; ++It) {
+ fprintf(stderr, "before replay %d\n", It);
+ run(N[It]);
+ if (Out != N[It]) {
+ fprintf(stderr, "replay %d: N %d (want %d)\n", It, Out, N[It]);
+ ++Errors;
+ }
+ }
+
+ // CHECK: before replay 1
+ // CHECK-NOT: reparametrizes for a changed literal argument
+ // CHECK: before replay 2
+ // CHECK: reparametrizes for a changed literal argument
+ // CHECK-NOT: reparametrizes for a changed literal argument
+ // CHECK-NOT: replay {{[0-9]+}}:
+ // CHECK: taskgraph firstprivate reparam errors: 0
+ fprintf(stderr, "taskgraph firstprivate reparam errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_lambda_capture.cpp b/offload/test/offloading/taskgraph_lambda_capture.cpp
new file mode 100644
index 00000000000000..16959030e13482
--- /dev/null
+++ b/offload/test/offloading/taskgraph_lambda_capture.cpp
@@ -0,0 +1,63 @@
+// RUN: %libomptarget-compilexx-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_INFO=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A lambda capturing by reference, called in a target region, is mapped as the
+// lambda object plus one fix-up entry per capture: the runtime writes the
+// captured variable's device address into the device copy of the lambda, which
+// otherwise holds the host address. The data plan has nothing to perform that
+// write with, so a graph containing one must be replayed on the host -- and say
+// so -- rather than hand the kernel a lambda full of host pointers.
+//
+// The fix-up only matters where the target region cannot see the captured
+// variables itself, as in the templated forall below: where it can, clang maps
+// them as arguments of their own and the kernel uses those. The lambda is
+// static and taken by reference so the graph captures nothing from a stack
+// frame, which would otherwise make the backend decline it for relocation and
+// hide the missing fix-up.
+
+#include <cstdio>
+
+#define N 256
+
+template <typename BodyTy> __attribute__((noinline)) void forall(BodyTy &Body) {
+#pragma omp taskgraph
+ {
+#pragma omp target teams distribute parallel for
+ for (int I = 0; I < N; ++I)
+ Body(I);
+ }
+}
+
+int main() {
+ const int Iters = 3;
+ double A[N], C[N];
+ for (int I = 0; I < N; ++I) {
+ A[I] = I;
+ C[I] = 0;
+ }
+ static auto Body = [&](int I) { C[I] += A[I]; };
+
+ // CHECK: taskgraph {{[0-9]+}} replays on the host: argument {{[0-9]+}} of leaf 0 ({{.*}}) is a lambda's by-reference capture
+#pragma omp target data map(tofrom : C[0 : N]) map(to : A[0 : N])
+ {
+ for (int It = 0; It < Iters; ++It)
+ forall(Body);
+ }
+
+ int Errors = 0;
+ for (int I = 0; I < N; ++I)
+ if (C[I] != Iters * A[I])
+ ++Errors;
+
+ // CHECK: taskgraph lambda capture errors: 0
+ std::printf("taskgraph lambda capture errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_map_always_borrowed.c b/offload/test/offloading/taskgraph_map_always_borrowed.c
new file mode 100644
index 00000000000000..fa9aef4da48f2f
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_always_borrowed.c
@@ -0,0 +1,74 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A map over a region that is already mapped from outside the graph performs no
+// copy: the reference count goes up rather than to one, and the enclosing
+// region owns the transfers. `always` is the clause that overrides that, and a
+// graph has to honour it per replay, because the host writes it exists to carry
+// happen between replays and the recording cannot know about them.
+//
+// The two arrays here differ only in the modifier, so B is the control: it
+// shows what the borrowed range does without `always` -- no copy in either
+// direction, so the host's own writes stay and the doubling never reaches it.
+// Asserting that too keeps the test honest, since a version of the runtime that
+// copied everything unconditionally would satisfy A's check alone.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 256
+#define ITERS 3
+
+int main(void) {
+ int *A = (int *)malloc(N * sizeof(int));
+ int *B = (int *)malloc(N * sizeof(int));
+ for (int I = 0; I < N; ++I)
+ A[I] = B[I] = 0;
+
+ int AlwaysErrors = 0, BorrowErrors = 0;
+
+ // Mapped out here, so both maps inside the graph borrow rather than map.
+#pragma omp target data map(tofrom : A[0 : N], B[0 : N])
+ {
+ for (int It = 0; It < ITERS; ++It) {
+ for (int I = 0; I < N; ++I) {
+ A[I] = 100 * (It + 1) + I;
+ B[I] = 100 * (It + 1) + I;
+ }
+
+#pragma omp taskgraph
+ {
+#pragma omp target map(always, tofrom : A[0 : N])
+ for (int I = 0; I < N; ++I)
+ A[I] *= 2;
+
+#pragma omp target map(tofrom : B[0 : N])
+ for (int I = 0; I < N; ++I)
+ B[I] *= 2;
+ }
+
+ for (int I = 0; I < N; ++I) {
+ if (A[I] != 2 * (100 * (It + 1) + I))
+ ++AlwaysErrors;
+ if (B[I] != 100 * (It + 1) + I)
+ ++BorrowErrors;
+ }
+ }
+ }
+
+ // CHECK: always over a borrowed range errors: 0
+ printf("always over a borrowed range errors: %d\n", AlwaysErrors);
+ // CHECK: borrowed without always, copies performed: 0
+ printf("borrowed without always, copies performed: %d\n", BorrowErrors);
+
+ free(B);
+ free(A);
+ return (AlwaysErrors + BorrowErrors) != 0;
+}
diff --git a/offload/test/offloading/taskgraph_map_attach.c b/offload/test/offloading/taskgraph_map_attach.c
new file mode 100644
index 00000000000000..697bc5aaf698c9
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_attach.c
@@ -0,0 +1,106 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// map(s.p[0:n]) is the shape that needs pointer attachment. The struct is
+// copied to the device with the host value of s.p still in it, so before a
+// kernel can follow that field it has to be overwritten with the device
+// address of the pointee. The runtime does that with a one-word transfer from
+// a staging buffer; a graph turns it into an ordinary memcpy node, which is
+// only correct if the staging buffer is stable for the life of the graph and
+// holds the right address at every replay.
+//
+// Two things move independently here, and the attachment has to survive both:
+// the struct, because the graph is replayed from a different frame, and the
+// pointee, because it is reallocated between replays. A dropped attachment
+// leaves a host address in the device struct, so it faults rather than
+// answering wrongly; a stale one reaches the previous replay's buffer.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 512
+
+// Tag is never read on the device: map(Local.P[0:N]) maps the bounding box of
+// the mapped members, which is P alone, so the rest of the struct is not there.
+struct S {
+ int *P;
+ int Tag;
+};
+
+static int *Data;
+
+static int work(int Seed) {
+ struct S Local;
+ Local.P = Data;
+ Local.Tag = Seed;
+
+ for (int I = 0; I < N; ++I)
+ Data[I] = Seed + I;
+
+ int Sum = 0;
+
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : Local.P[0 : N]) map(tofrom : Sum)
+ {
+ Sum = 0;
+ for (int I = 0; I < N; ++I) {
+ Local.P[I] += 1;
+ Sum += Local.P[I];
+ }
+ }
+ }
+
+ int Want = 0;
+ for (int I = 0; I < N; ++I)
+ Want += Seed + I + 1;
+
+ int Errors = (Sum != Want);
+ for (int I = 0; I < N; ++I)
+ if (Data[I] != Seed + I + 1)
+ ++Errors;
+ return Errors;
+}
+
+// Reach work() through a deeper chain each time so its struct sits at a
+// different address on every encounter of the one taskgraph directive in it.
+static int atDepth(int Depth, int Seed) {
+ char Pad[4096];
+ Pad[0] = (char)Depth;
+ int Errors = Depth ? atDepth(Depth - 1, Seed) : work(Seed);
+ return Errors + (Pad[0] != (char)Depth);
+}
+
+// Give the pointee a new address, so the pointer the graph recorded is stale
+// even though the struct holding it is not.
+static int moved(int Seed) {
+ int *Old = Data;
+ Data = (int *)malloc(N * sizeof(int));
+ int Errors = work(Seed);
+ free(Old);
+ return Errors;
+}
+
+int main(void) {
+ Data = (int *)malloc(N * sizeof(int));
+
+ int Errors = 0;
+ Errors += atDepth(0, 1);
+ Errors += atDepth(2, 3);
+ Errors += atDepth(5, 7);
+ Errors += moved(11);
+ Errors += moved(13);
+
+ free(Data);
+
+ // CHECK: taskgraph map attach errors: 0
+ printf("taskgraph map attach errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_map_close.c b/offload/test/offloading/taskgraph_map_close.c
new file mode 100644
index 00000000000000..854c32f174cd9a
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_close.c
@@ -0,0 +1,73 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env HSA_XNACK=1 LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env HSA_XNACK=1 %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// REQUIRES: unified_shared_memory
+
+// Under unified shared memory a map works on the host allocation unless it
+// carries `close`, which asks for a device copy instead. The difference is
+// only visible through a map that does not copy back: a kernel writing through
+// a `to`-only map changes the host array when the map works on host memory, and
+// leaves it untouched when it had a device copy of its own. So a graph that
+// ignored `close` would still get the `tofrom` arrays right and silently leak
+// every write through the `close, to` one. The host path (plugin backend
+// off) must agree with the plugin path.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#pragma omp requires unified_shared_memory
+
+int main() {
+ const int N = 1024;
+ const int Iters = 4;
+ int *CloseTo = (int *)malloc(N * sizeof(int));
+ int *CloseToFrom = (int *)malloc(N * sizeof(int));
+ int *SharedTo = (int *)malloc(N * sizeof(int));
+ int *SharedToFrom = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ CloseTo[i] = CloseToFrom[i] = SharedTo[i] = SharedToFrom[i] = i;
+
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(close, to : CloseTo[0 : N]) \
+ map(close, tofrom : CloseToFrom[0 : N]) map(to : SharedTo[0 : N]) \
+ map(tofrom : SharedToFrom[0 : N])
+ {
+ for (int i = 0; i < N; ++i) {
+ CloseTo[i] += 1;
+ CloseToFrom[i] += 1;
+ SharedTo[i] += 1;
+ SharedToFrom[i] += 1;
+ }
+ }
+ }
+ }
+
+ // Each encounter maps CloseTo afresh from the unchanged host array and throws
+ // the device copy away, so the host never sees an update. Every other array
+ // accumulates one per encounter: CloseToFrom through its copy back, and the
+ // two maps without `close` because the kernel wrote the host memory itself.
+ int errors = 0;
+ for (int i = 0; i < N; ++i) {
+ if (CloseTo[i] != i)
+ ++errors;
+ if (CloseToFrom[i] != i + Iters || SharedTo[i] != i + Iters ||
+ SharedToFrom[i] != i + Iters)
+ ++errors;
+ }
+
+ // CHECK: taskgraph map close errors: 0
+ printf("taskgraph map close errors: %d\n", errors);
+ free(CloseTo);
+ free(CloseToFrom);
+ free(SharedTo);
+ free(SharedToFrom);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_map_drift.c b/offload/test/offloading/taskgraph_map_drift.c
new file mode 100644
index 00000000000000..b51e35919f3083
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_drift.c
@@ -0,0 +1,115 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_TASKGRAPH_REPORT_DRIFT=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=DRIFT
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=QUIET
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A graph is recorded once and replayed many times, but the mapping state it
+// was recorded against belongs to the program, not to the graph, and can differ
+// on every replay. A region the graph found mapped may be gone, and one it
+// found unmapped may have acquired a mapping, so whether the graph should use
+// somebody else's device buffer or one of its own is a question that has to be
+// re-asked per replay rather than settled once when the graph is built.
+//
+// An `if` clause on an enclosing `target data` is what varies it without
+// varying the graph: the construct, and so the graph, is the same one every
+// iteration, while the mapping around it comes and goes.
+//
+// Both directions of the change are wrong in the same way -- two device copies
+// of one host range -- but they fail through different paths. Where the graph
+// believes a region is unmapped but it is now mapped, the graph copies through a
+// buffer of its own while the enclosing region holds a second copy that never
+// sees the kernel, and the enclosing region's copy-out then writes that older
+// copy back over the host. Where the graph believes a region is mapped but the
+// mapping has since gone, the graph is left addressing a device allocation that
+// was freed underneath it.
+//
+// In the first direction, a region unmapped when the graph is finalized, and
+// balanced within it, is one the graph keeps to itself, in storage of its own
+// with no mapping-table entry. Finding it mapped from outside on a later
+// replay, by a mapping that covers it, the graph borrows that mapping for the
+// replay as it would a region it found mapped: the device address is the
+// mapping's, and the graph's own copies are off, the mapping's standing in for
+// them. Nothing is declined or warned about.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 1024
+#define ITERS 3
+
+int main() {
+ int *A = (int *)malloc(N * sizeof(int));
+ int *B = (int *)malloc(N * sizeof(int));
+ for (int I = 0; I < N; ++I)
+ A[I] = B[I] = I;
+
+ // Unmapped when the graph is built, mapped on every replay.
+ for (int It = 0; It < ITERS; ++It) {
+#pragma omp target data map(tofrom : A[0 : N]) if (It > 0)
+ {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A[0 : N])
+ for (int I = 0; I < N; ++I)
+ A[I] += 1;
+ }
+ }
+ }
+
+ int AErrors = 0;
+ for (int I = 0; I < N; ++I)
+ if (A[I] != I + ITERS)
+ ++AErrors;
+ // CHECK: unmapped when recorded errors: 0
+ printf("unmapped when recorded errors: %d\n", AErrors);
+
+ // Mapped when the graph is built, unmapped on every replay.
+ for (int It = 0; It < ITERS; ++It) {
+#pragma omp target data map(tofrom : B[0 : N]) if (It == 0)
+ {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : B[0 : N])
+ for (int I = 0; I < N; ++I)
+ B[I] += 1;
+ }
+ }
+ }
+
+ int BErrors = 0;
+ for (int I = 0; I < N; ++I)
+ if (B[I] != I + ITERS)
+ ++BErrors;
+ // CHECK: mapped when recorded errors: 0
+ printf("mapped when recorded errors: %d\n", BErrors);
+
+ // CHECK: taskgraph map drift errors: 0
+ printf("taskgraph map drift errors: %d\n", AErrors + BErrors);
+
+ // Right answers alone would not show the drift was taken, so the count is
+ // pinned too. Three, not four: the first loop's two replays each follow a
+ // mapping appearing, but the second loop's mapping goes away once and then
+ // stays away, so its later replay finds the state it left behind and the
+ // check costs nothing. A fourth would mean that short-circuit is broken.
+ // DRIFT-COUNT-3: mapping drift, re-resolving ranges
+ // DRIFT-NOT: mapping drift
+ //
+ // Every replay runs on the device.
+ // QUIET-NOT: omptarget message
+ free(B);
+ free(A);
+ return (AErrors + BErrors) != 0;
+}
diff --git a/offload/test/offloading/taskgraph_map_dynamic_bound_saved.c b/offload/test/offloading/taskgraph_map_dynamic_bound_saved.c
new file mode 100644
index 00000000000000..f287e3f6cace92
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_dynamic_bound_saved.c
@@ -0,0 +1,112 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_TASKGRAPH_REPORT_DRIFT=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=RELOC
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// Two things that pull in opposite directions, in one region.
+//
+// `map(A[Lo : LEN])` names a section at a position that is not a constant, so
+// the recorded slot is a host address with no compile-time relationship to the
+// variable it came from. Re-deriving it is a runtime delta: where the variable
+// is now, plus the distance the recording had between the variable and the
+// section. A replay from a different frame is what makes the difference
+// visible, so the region is reached through a deeper call chain each time.
+//
+// `firstprivate(saved : Coef)` beside it must *not* move. Its slot points at a
+// snapshot inside the recorded blob -- that is what `saved` means -- so
+// relocating it back to the live variable is precisely the thing to avoid,
+// while the map next to it still has to be relocated. Coef is refilled per
+// encounter so a replay that read the live copy would be caught by value.
+//
+// Values outside the window are checked too: an offset re-derived a row out
+// would write over them and still leave the window looking right.
+
+#include <stdio.h>
+
+#define N 512
+#define LEN 128
+#define M 8
+
+// Not a constant to the compiler, and the same on every encounter: what varies
+// between replays is the frame, not the section.
+static int Lo;
+
+static void *Seen[3];
+static int Encounter;
+
+static int work(int Seed) {
+ int A[N];
+ int Coef[M];
+ for (int I = 0; I < N; ++I)
+ A[I] = Seed + I;
+ for (int J = 0; J < M; ++J)
+ Coef[J] = Seed + J;
+
+ Seen[Encounter++] = A;
+
+#pragma omp taskgraph
+ {
+#pragma omp target firstprivate(saved : Coef) map(tofrom : A[Lo : LEN])
+ for (int I = 0; I < LEN; ++I)
+ A[Lo + I] += Coef[I % M];
+ }
+
+ int Errors = 0;
+ for (int I = 0; I < N; ++I) {
+ int Want = Seed + I;
+ // The first encounter is the one that recorded, so its Coef is the one
+ // every replay privatizes from.
+ if (I >= Lo && I < Lo + LEN)
+ Want += 100 + ((I - Lo) % M);
+ if (A[I] != Want)
+ ++Errors;
+ }
+ return Errors;
+}
+
+// Reach work() through a deeper chain each time, so its locals -- and with them
+// both the variable the section is measured from and the frame a stale slot
+// would write into -- sit somewhere else on every encounter. The padding is
+// written and read so it cannot be elided.
+static int atDepth(int Depth, int Seed) {
+ char Pad[4096];
+ Pad[0] = (char)Depth;
+ int Errors = Depth ? atDepth(Depth - 1, Seed) : work(Seed);
+ return Errors + (Pad[0] != (char)Depth);
+}
+
+int main(int argc, char **argv) {
+ Lo = 64 + (argc - 1);
+
+ int Errors = 0;
+ Errors += atDepth(0, 100);
+ Errors += atDepth(3, 200);
+ Errors += atDepth(7, 300);
+
+ // CHECK: distinct frames: 3
+ int Distinct =
+ 1 + (Seen[1] != Seen[0]) + (Seen[2] != Seen[0] && Seen[2] != Seen[1]);
+ printf("distinct frames: %d\n", Distinct);
+
+ // CHECK: taskgraph dynamic bound with saved firstprivate errors: 0
+ printf("taskgraph dynamic bound with saved firstprivate errors: %d\n",
+ Errors);
+
+ // Right answers alone would not show the section was re-derived rather than
+ // the frames happening to line up, so the count is pinned: one recording
+ // encounter and two replays that each relocate.
+ // RELOC-COUNT-2: relocated host captures, re-resolving ranges
+ // RELOC-NOT: relocated host captures
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_map_partial_borrow.c b/offload/test/offloading/taskgraph_map_partial_borrow.c
new file mode 100644
index 00000000000000..6d0044ce45ce53
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_partial_borrow.c
@@ -0,0 +1,81 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-fail-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefix=NOGRAPH
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-fail-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefix=GRAPH
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A mapped range inside a taskgraph that only *partially* overlaps one mapped
+// outside it. Asking whether a range is mapped is a question about whole-range
+// containment, so a partial overlap answers no, and a graph that reads that as
+// "not mapped" takes the range for its own and gives it a buffer -- one which
+// aliases the outer mapping's device memory over the part they share, with
+// nothing keeping the two coherent. Nor is there an address to borrow instead,
+// since the mapping does not cover the whole range.
+//
+// So there is no correct silent answer, and libomptarget does not attempt one:
+// outside a graph this shape is refused as an explicit extension of mapped
+// data. The graph declines the replay rather than guessing, which hands it to
+// the host path and so to that same refusal. Both runs below therefore expect
+// to die, of the same error, and the point of running both is that the graph
+// refuses the same program the plain path refuses.
+//
+// The range is unmapped when the graph is finalized and balanced within it, so
+// the graph keeps it on the device for itself rather than in the mapping table.
+// The outer mapping is made between replays, when the graph holds nothing, and
+// the replay then finds its range partly mapped and declines.
+//
+// The overlap is arranged to arrive between replays rather than at record time.
+// A partial overlap present while the graph is recorded is already caught, by
+// libomptarget itself, on the recording execution -- that pass maps normally --
+// so the graph never reaches the point of resolving it. It is the mapping that
+// appears afterwards that a graph resolving its ranges once would miss.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 1024
+#define OUTER (N / 2) // outer maps [0, 512)
+#define INNER_LO (N / 4) // graph maps [256, 768): overlaps the outer tail
+#define ITERS 3
+
+int main() {
+ int *A = (int *)malloc(N * sizeof(int));
+ for (int I = 0; I < N; ++I)
+ A[I] = I;
+
+ for (int It = 0; It < ITERS; ++It) {
+ // Absent while the graph is recorded, so the graph takes its range for its
+ // own, and present for the replays.
+#pragma omp target data map(tofrom : A[0 : OUTER]) if (It > 0)
+ {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A[INNER_LO : OUTER])
+ for (int I = 0; I < OUTER; ++I)
+ A[INNER_LO + I] += 1;
+ }
+ }
+ }
+
+ // NOGRAPH: explicit extension not allowed
+ // NOGRAPH: fatal error
+ // GRAPH-NOT: overlaps memory
+ // GRAPH: taskgraph keeps [{{.*}}) on the device for itself, and it is now partly mapped outside the graph
+ // GRAPH: taskgraph {{[0-9]+}} replays on the host this time
+ // GRAPH: explicit extension not allowed
+ // GRAPH: fatal error
+ printf("reached the end\n");
+ // NOGRAPH-NOT: reached the end
+ // GRAPH-NOT: reached the end
+ free(A);
+ return 0;
+}
diff --git a/offload/test/offloading/taskgraph_map_private.c b/offload/test/offloading/taskgraph_map_private.c
new file mode 100644
index 00000000000000..9872a65f4817da
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_private.c
@@ -0,0 +1,122 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Privatised arguments want device storage that is not a mapping: nothing else
+// in the program has a view of it, so it is absent from the mapping table.
+// Outside a graph the runtime allocates it per launch, fills it and frees it
+// again; a graph cannot, because the address is written into a kernarg image
+// when the graph is lowered. So the plan allocates once for the graph and
+// refills the contents each replay.
+//
+// Both shapes that reach this are exercised, and neither is a corner case:
+//
+// - map(p[0:n]) on a plain pointer. `p` is predetermined firstprivate on the
+// construct and its private copy is initialised by
+// corresponding-pointer-initialization -- the *device* address of the
+// pointee, which is only known once the pointee's own map is resolved. A
+// stale copy points the kernel at host memory.
+// - firstprivate(S) on an aggregate, with no `saved` modifier, so the copy is
+// of the variable as it stands at this replay. Storage reused from the last
+// replay without a refill would silently hand the kernel the previous
+// encounter's value.
+//
+// Each is replayed from a different frame, which is what makes the refill read
+// the live variable rather than the recorded address.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 512
+
+struct Agg {
+ int A[4];
+ double D;
+};
+
+static int *Data;
+
+static int pointerCase(int Seed) {
+ int *P = Data;
+ for (int I = 0; I < N; ++I)
+ P[I] = Seed + I;
+
+ int Sum = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : P[0 : N]) map(tofrom : Sum)
+ {
+ Sum = 0;
+ for (int I = 0; I < N; ++I) {
+ P[I] += 1;
+ Sum += P[I];
+ }
+ }
+ }
+
+ int Want = 0;
+ for (int I = 0; I < N; ++I)
+ Want += Seed + I + 1;
+
+ int Errors = (Sum != Want);
+ for (int I = 0; I < N; ++I)
+ if (Data[I] != Seed + I + 1)
+ ++Errors;
+ return Errors;
+}
+
+static int aggregateCase(int Seed) {
+ struct Agg S;
+ for (int I = 0; I < 4; ++I)
+ S.A[I] = Seed + I;
+ S.D = Seed;
+
+ int Sum = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target firstprivate(S) map(tofrom : Sum)
+ {
+ Sum = (int)S.D;
+ for (int I = 0; I < 4; ++I)
+ Sum += S.A[I];
+ }
+ }
+
+ int Want = Seed;
+ for (int I = 0; I < 4; ++I)
+ Want += Seed + I;
+ return Sum != Want;
+}
+
+// Reach the case through a deeper chain each time, so its locals sit at a
+// different address on every encounter of the taskgraph directive inside it.
+static int atDepth(int Depth, int Seed, int (*Fn)(int)) {
+ char Pad[4096];
+ Pad[0] = (char)Depth;
+ int Errors = Depth ? atDepth(Depth - 1, Seed, Fn) : Fn(Seed);
+ return Errors + (Pad[0] != (char)Depth);
+}
+
+int main(void) {
+ Data = (int *)malloc(N * sizeof(int));
+
+ int Errors = 0;
+ Errors += atDepth(0, 1, pointerCase);
+ Errors += atDepth(2, 3, pointerCase);
+ Errors += atDepth(5, 7, pointerCase);
+ Errors += atDepth(0, 10, aggregateCase);
+ Errors += atDepth(3, 20, aggregateCase);
+ Errors += atDepth(6, 30, aggregateCase);
+
+ free(Data);
+
+ // CHECK: taskgraph map private errors: 0
+ printf("taskgraph map private errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_map_relocated_mixed.c b/offload/test/offloading/taskgraph_map_relocated_mixed.c
new file mode 100644
index 00000000000000..2e7bd302e58362
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_relocated_mixed.c
@@ -0,0 +1,115 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 -DMODE=0 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 -DMODE=0 && \
+// RUN: env LIBOMPTARGET_TASKGRAPH_REPORT_DRIFT=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefixes=CHECK,RELOC
+//
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 -DMODE=1 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic --check-prefix=HOST1
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 -DMODE=1 && \
+// RUN: %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefixes=OVERLAP,DECLINE,HOST1
+//
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 -DMODE=2 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic --check-prefix=HOST2
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 -DMODE=2 && \
+// RUN: %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefixes=PARTIAL,DECLINE,HOST2
+//
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 -DMODE=3 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic --check-prefix=HOST3
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 -DMODE=3 && \
+// RUN: env LIBOMPTARGET_TASKGRAPH_REPORT_DRIFT=1 \
+// RUN: %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefixes=OVERLAP,DECLINE,AFTER,HOST3
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A graph's ranges follow its captures when a replay relocates them, and they
+// need not all move by one distance: a pointer parameter's section moves to
+// wherever the pointer now points, independently of the frame. What a range
+// may not do is come apart or run into another one, because the grouping is
+// fixed when the graph is recorded.
+//
+// MODE=0 P and Q move by different distances, and one onto the other's
+// recorded address; the graph follows both.
+// MODE=1 P and Q were separate regions when recorded and name one region
+// on the replay: refused as an overlap.
+// MODE=2 P and Q were one region when recorded and name two on the
+// replay: refused as moving part of a region.
+// MODE=3 MODE=1, then a replay the graph can perform again, which has to
+// reach the device with nothing left over from the declined one.
+//
+// A refused replay is declined to the host, which re-runs the mappers and so
+// handles all three; the results are checked against it.
+
+#include <stdio.h>
+
+#define N 256
+
+static int Buf[3][N];
+
+static void run(int *P, int *Q) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : P[0 : N]) map(tofrom : Q[0 : N])
+ for (int I = 0; I < N; ++I) {
+ P[I] += 1;
+ Q[I] += 2;
+ }
+ }
+}
+
+static int uniform(int K) {
+ for (int I = 1; I < N; ++I)
+ if (Buf[K][I] != Buf[K][0])
+ return 0;
+ return 1;
+}
+
+int main(void) {
+#if MODE == 0
+ run(Buf[0], Buf[1]);
+ run(Buf[2], Buf[0]);
+ run(Buf[1], Buf[2]);
+#elif MODE == 1
+ run(Buf[0], Buf[1]);
+ run(Buf[2], Buf[2]);
+#elif MODE == 2
+ run(Buf[0], Buf[0]);
+ run(Buf[1], Buf[2]);
+#else
+ run(Buf[0], Buf[1]);
+ run(Buf[2], Buf[2]);
+ run(Buf[1], Buf[2]);
+#endif
+
+ // What a refused replay reports, ahead of the results on stdout.
+ // OVERLAP: taskgraph relocation put [{{.*}}) over [{{.*}}), which were separate regions when the graph was recorded
+ // PARTIAL: taskgraph relocation moved part of the region [{{.*}}) without moving the rest of it
+ // DECLINE: taskgraph {{[0-9]+}} replays on the host this time
+ // AFTER-NOT: replays on the host
+ // AFTER: relocated host captures, re-resolving ranges
+ // AFTER-NOT: replays on the host
+
+ // The first call records and the two after it each relocate.
+ // RELOC-COUNT-2: relocated host captures, re-resolving ranges
+ // RELOC-NOT: relocated host captures
+
+ // CHECK: buffers: 3 3 3 uniform 1
+ // HOST1: buffers: 1 2 3 uniform 1
+ // HOST2: buffers: 3 1 2 uniform 1
+ // HOST3: buffers: 1 3 5 uniform 1
+ printf("buffers: %d %d %d uniform %d\n", Buf[0][0], Buf[1][0], Buf[2][0],
+ uniform(0) && uniform(1) && uniform(2));
+ return 0;
+}
diff --git a/offload/test/offloading/taskgraph_map_relocated_stack.c b/offload/test/offloading/taskgraph_map_relocated_stack.c
new file mode 100644
index 00000000000000..56e63e020b8206
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_relocated_stack.c
@@ -0,0 +1,90 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_TASKGRAPH_REPORT_DRIFT=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic --check-prefix=RELOC
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A recorded target region keeps the map arrays it was launched with, so the
+// host addresses in them are the ones the mapped variables had on the
+// encounter that recorded the graph. Nothing says a later encounter runs from
+// the same frame: the same function reached through a different call chain
+// puts its locals somewhere else entirely, and replaying against the recorded
+// addresses would then read and write a dead frame.
+//
+// clang emits a relocation callback beside the recorded region, the runtime
+// calls it with the taskgraph record reconstructed for this encounter, and it
+// rewrites the map arrays; libomptarget then follows its ranges to wherever
+// they landed. Without any part of that the graph writes its result into the
+// recording frame and the array checked here is left untouched, so the value
+// check below is the real test -- the address report only shows that the three
+// encounters genuinely ran from different frames.
+
+#include <stdio.h>
+
+#define N 1024
+
+static void *Seen[3];
+static int Encounter;
+
+static int work(int Seed) {
+ int A[N];
+ for (int I = 0; I < N; ++I)
+ A[I] = Seed + I;
+
+ Seen[Encounter++] = A;
+
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : A[0 : N])
+ for (int I = 0; I < N; ++I)
+ A[I] += 1;
+ }
+
+ int Errors = 0;
+ for (int I = 0; I < N; ++I)
+ if (A[I] != Seed + I + 1)
+ ++Errors;
+ return Errors;
+}
+
+// Reach work() through a deeper chain each time, so its locals sit at a
+// different address on every encounter of the one taskgraph directive inside
+// it. The padding is written and read so it cannot be elided.
+static int atDepth(int Depth, int Seed) {
+ char Pad[4096];
+ Pad[0] = (char)Depth;
+ int Errors = Depth ? atDepth(Depth - 1, Seed) : work(Seed);
+ return Errors + (Pad[0] != (char)Depth);
+}
+
+int main(void) {
+ int Errors = 0;
+ Errors += atDepth(0, 100);
+ Errors += atDepth(3, 200);
+ Errors += atDepth(7, 300);
+
+ // CHECK: distinct frames: 3
+ int Distinct =
+ 1 + (Seen[1] != Seen[0]) + (Seen[2] != Seen[0] && Seen[2] != Seen[1]);
+ printf("distinct frames: %d\n", Distinct);
+
+ // CHECK: taskgraph relocated stack errors: 0
+ printf("taskgraph relocated stack errors: %d\n", Errors);
+
+ // Right answers alone would not show relocation was taken rather than the
+ // frames happening to line up, so the count is pinned: the first encounter
+ // records and the two after it each replay from a frame of their own.
+ // RELOC-COUNT-2: relocated host captures, re-resolving ranges
+ // RELOC-NOT: relocated host captures
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_map_struct_member.c b/offload/test/offloading/taskgraph_map_struct_member.c
new file mode 100644
index 00000000000000..276644189e8779
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_struct_member.c
@@ -0,0 +1,62 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Mapping through a struct member, which is the shape that produces member-of
+// and pointer-and-pointee map entries plus the attachments that go with them.
+// Nothing about it needs a lowering category of its own: the plan's range
+// gathering already puts a member inside the range covering its struct, which
+// is exactly the coherence the two need, and the kernel-argument arithmetic
+// follows from the same gathering. What the shape does need is relocation,
+// since the base of `s.a[0:N]` is what `s.a` holds and that is loaded afresh on
+// every replay.
+//
+// Both `s.a` and `s.b` are mapped, so the graph carries two attachments over one
+// struct; getting either wrong points the kernel at host memory.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+struct Buffers {
+ int *a;
+ int *b;
+};
+
+int main() {
+ const int N = 1024;
+ const int Iters = 3;
+ struct Buffers s;
+ s.a = (int *)malloc(N * sizeof(int));
+ s.b = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i) {
+ s.a[i] = i;
+ s.b[i] = 2 * i;
+ }
+
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : s.a[0 : N]) map(to : s.b[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ s.a[i] = s.a[i] + s.b[i];
+ }
+ }
+ }
+
+ // Each encounter adds b[i] == 2 * i, so a[i] == i + Iters * 2 * i.
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (s.a[i] != i + Iters * 2 * i)
+ ++errors;
+
+ // CHECK: taskgraph struct member errors: 0
+ printf("taskgraph struct member errors: %d\n", errors);
+ free(s.b);
+ free(s.a);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_present_absent.c b/offload/test/offloading/taskgraph_present_absent.c
new file mode 100644
index 00000000000000..edc764a0fa2464
--- /dev/null
+++ b/offload/test/offloading/taskgraph_present_absent.c
@@ -0,0 +1,71 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %not --crash env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %not --crash %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --check-prefixes=CHECK,PLUGIN
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// `present` says the region is expected to be mapped already and this map must
+// not bring it in. A graph cannot settle that at record time: whether a region
+// is mapped belongs to the program's state, so the encounter that recorded may
+// find it there and a replay may not. Getting it wrong is silent -- the graph
+// would allocate a buffer of its own, and then two device copies of one host
+// range exist and the program gets plausible wrong answers -- so the required
+// behaviour is a runtime error, per replay, on whichever path is running.
+//
+// The mapping is therefore made only for the recording encounter and deleted
+// after it, so the graph is recorded against a present range and replayed
+// against an absent one. Both paths report and abort: the host path re-runs
+// the ordinary map machinery and gets the usual diagnostic, while the plugin
+// path fails resolving the range before it submits anything, says so, and
+// declines the replay to the host path, which then aborts the same way.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 256
+
+int main(void) {
+ int *A = (int *)malloc(N * sizeof(int));
+ for (int I = 0; I < N; ++I)
+ A[I] = I;
+
+ for (int It = 0; It < 2; ++It) {
+ if (It == 0) {
+#pragma omp target enter data map(to : A[0 : N])
+ }
+
+ // CHECK: encounter 0
+ // CHECK: encounter 1
+ printf("encounter %d\n", It);
+ fflush(stdout);
+
+#pragma omp taskgraph
+ {
+#pragma omp target map(present, tofrom : A[0 : N])
+ for (int I = 0; I < N; ++I)
+ A[I] += 1;
+ }
+
+ if (It == 0) {
+#pragma omp target exit data map(delete : A[0 : N])
+ }
+ }
+
+ // PLUGIN: taskgraph requires {{.*}} to be present, and it is not
+ // PLUGIN: taskgraph {{[0-9]+}} replays on the host this time
+ // CHECK: 'present' map type modifier does not exist
+
+ // CHECK-NOT: reached the end
+ printf("reached the end\n");
+ free(A);
+ return 0;
+}
diff --git a/offload/test/offloading/taskgraph_present_replay.c b/offload/test/offloading/taskgraph_present_replay.c
new file mode 100644
index 00000000000000..858bfa10299212
--- /dev/null
+++ b/offload/test/offloading/taskgraph_present_replay.c
@@ -0,0 +1,53 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A replayed 'omp taskgraph' whose 'target' operates on data kept live by an
+// enclosing 'target data' region. The data is present on the device for the
+// whole replay, so each encounter must accumulate into the *same* live device
+// allocation: no per-replay copy from the (stale) host, no premature copy-back.
+// The HIP-Graph backend must bake the live device address and elide the
+// implicit map copies; a wrong (graph-owned-buffer + per-replay H2D/D2H) model
+// would re-seed from the never-updated host array and yield i + 1 instead of
+// i + Iters.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 1024;
+ const int Iters = 4;
+ int *a = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = i;
+
+#pragma omp target data map(tofrom : a[0 : N])
+ {
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : a[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] + 1;
+ }
+ }
+ }
+ } // end target data: copies the accumulated device values back to the host.
+
+ // a was present throughout; each of the Iters encounters adds 1 in place, so
+ // a[i] == i + Iters.
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (a[i] != i + Iters)
+ ++errors;
+
+ // CHECK: taskgraph present replay errors: 0
+ printf("taskgraph present replay errors: %d\n", errors);
+ free(a);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_decline.c b/offload/test/offloading/taskgraph_target_decline.c
new file mode 100644
index 00000000000000..e89566a7e6a433
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_decline.c
@@ -0,0 +1,61 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A graph a device backend is expected to decline, replayed with the backend
+// enabled. A user-defined mapper expands to an entry list the *runtime*
+// computes: the generated mapper emits an optional leading entry, then one
+// component list per element of the section it was invoked on, so how many map
+// entries there are is not a property of the recording. A graph's topology is
+// fixed when it is recorded, so the plan declines and libomp replays the graph
+// on the host, issuing each recorded target region as its dataflow walk reaches
+// it -- and the mapper then runs exactly as it does outside a graph.
+//
+// Declining is a supported outcome, not a failure: the point of this test is
+// that the answer is still right, so a backend that grows support for these maps
+// and stops declining keeps passing.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+struct Vec {
+ int *P;
+ int Len;
+};
+
+#pragma omp declare mapper(struct Vec v) map(v, v.P[0 : v.Len])
+
+int main() {
+ const int N = 1024;
+ const int Iters = 3;
+ struct Vec V;
+ V.P = (int *)malloc(N * sizeof(int));
+ V.Len = N;
+ for (int i = 0; i < N; ++i)
+ V.P[i] = i;
+
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : V)
+ {
+ for (int i = 0; i < V.Len; ++i)
+ V.P[i] = V.P[i] + 1;
+ }
+ }
+ }
+
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (V.P[i] != i + Iters)
+ ++errors;
+
+ // CHECK: taskgraph decline errors: 0
+ printf("taskgraph decline errors: %d\n", errors);
+ free(V.P);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_unordered_copy_decline.c b/offload/test/offloading/taskgraph_unordered_copy_decline.c
new file mode 100644
index 00000000000000..75a787a8cc62e2
--- /dev/null
+++ b/offload/test/offloading/taskgraph_unordered_copy_decline.c
@@ -0,0 +1,60 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_INFO=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A `target enter data` and a `target` region reading the same data, with
+// nothing ordering the two. Outside a graph, whichever maps first copies the
+// data in, and a map is atomic, so the kernel always sees it. The data plan
+// settles which leaf copies when the graph is built, and the region tree is all
+// that orders one leaf against another, so here the kernel could read the data
+// mid-copy, or before it. The graph maps A itself on every replay, so the copy
+// always runs, and every replay is declined to the host, where the results are
+// right. The decline is reported once. taskgraph_unordered_copy_replay.c has
+// replays that keep the device graph.
+//
+// The host changes A before every replay, and the kernel reads the end of it,
+// which a copy reaches last.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N (1024 * 1024)
+#define M 64
+#define ITERS 4
+
+int main(void) {
+ int *A = (int *)malloc(N * sizeof(int));
+ int B[M];
+ int DepA, DepB, Errors = 0;
+
+ // CHECK: taskgraph {{[0-9]+}} replays on the host: leaf {{[0-9]+}} copies [{{.*}}) and leaf {{[0-9]+}} uses it, and nothing orders the two
+ for (int It = 0; It < ITERS; ++It) {
+ for (int I = 0; I < N; ++I)
+ A[I] = 1000 * It + I % 7;
+#pragma omp taskgraph
+ {
+#pragma omp target map(to : A[0 : N]) map(from : B[0 : M]) \
+ nowait depend(out : DepB)
+ for (int I = 0; I < M; ++I)
+ B[I] = A[N - M + I];
+#pragma omp target enter data map(to : A[0 : N]) nowait depend(out : DepA)
+#pragma omp target exit data map(release : A[0 : N]) \
+ nowait depend(in : DepA, DepB)
+ }
+#pragma omp taskwait
+ for (int I = 0; I < M; ++I)
+ Errors += B[I] != 1000 * It + (N - M + I) % 7;
+ }
+
+ // CHECK: taskgraph unordered copy decline errors: 0
+ printf("taskgraph unordered copy decline errors: %d\n", Errors);
+ free(A);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_unordered_copy_replay.c b/offload/test/offloading/taskgraph_unordered_copy_replay.c
new file mode 100644
index 00000000000000..3213788957466c
--- /dev/null
+++ b/offload/test/offloading/taskgraph_unordered_copy_replay.c
@@ -0,0 +1,81 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_INFO=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// Two unordered `target` regions each mapping all of T `tofrom` and writing
+// half of it. When the graph maps T itself, each copies T in and out, and
+// nothing orders those copies against the other kernel, so the replay goes to
+// the host. When the program has mapped T around the replay, the graph copies
+// nothing and keeps the device graph. The decision is the replay's: the same
+// graph goes to the host and comes back as the program maps and unmaps T, and
+// says so once each time it starts going to the host.
+//
+// The markers go to stderr, as the runtime's messages do, so that the two
+// interleave in order.
+
+#include <stdio.h>
+
+#define N 4096
+#define ITERS 7
+
+int main(void) {
+ int T[N];
+ int Dep1, Dep2, Errors = 0;
+ for (int I = 0; I < N; ++I)
+ T[I] = 0;
+
+ for (int It = 0; It < ITERS; ++It) {
+ const int Mapped = It < 3 || It == 6;
+ // The first encounter records; replays start at 1.
+ if (It > 0)
+ fprintf(stderr, "replay %d %s\n", It, Mapped ? "mapped" : "unmapped");
+ if (Mapped) {
+#pragma omp target enter data map(to : T[0 : N])
+ }
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : T[0 : N]) nowait depend(out : Dep1)
+ for (int I = 0; I < N / 2; ++I)
+ T[I] += 1;
+#pragma omp target map(tofrom : T[0 : N]) nowait depend(out : Dep2)
+ for (int I = N / 2; I < N; ++I)
+ T[I] += 2;
+ }
+#pragma omp taskwait
+ if (Mapped) {
+#pragma omp target exit data map(from : T[0 : N])
+ }
+ for (int I = 0; I < N; ++I)
+ Errors += T[I] != (I < N / 2 ? 1 : 2) * (It + 1);
+ }
+
+ // A replay on the host enters each kernel through the runtime, which says so
+ // under LIBOMPTARGET_INFO; one on the device does not.
+ //
+ // CHECK-NOT: replays on the host
+ // CHECK: replay 1 mapped
+ // CHECK-NOT: {{Entering OpenMP kernel|replays on the host}}
+ // CHECK: replay 2 mapped
+ // CHECK-NOT: {{Entering OpenMP kernel|replays on the host}}
+ // CHECK: replay 3 unmapped
+ // CHECK-NEXT: taskgraph {{[0-9]+}} replays on the host: leaf {{[0-9]+}} copies [{{.*}}) and leaf {{[0-9]+}} uses it, and nothing orders the two
+ // CHECK: Entering OpenMP kernel
+ // CHECK: replay 4 unmapped
+ // CHECK-NOT: replays on the host
+ // CHECK: Entering OpenMP kernel
+ // CHECK: replay 5 unmapped
+ // CHECK-NOT: replays on the host
+ // CHECK: Entering OpenMP kernel
+ // CHECK: replay 6 mapped
+ // CHECK-NOT: {{Entering OpenMP kernel|replays on the host}}
+ // CHECK: taskgraph unordered copy replay errors: 0
+ fprintf(stderr, "taskgraph unordered copy replay errors: %d\n", Errors);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_unordered_shared.c b/offload/test/offloading/taskgraph_unordered_shared.c
new file mode 100644
index 00000000000000..82d0cad0b5eb60
--- /dev/null
+++ b/offload/test/offloading/taskgraph_unordered_shared.c
@@ -0,0 +1,73 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_INFO=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --implicit-check-not='replays on the host'
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Unordered leaves sharing data that the graph keeps on the device, unlike
+// taskgraph_unordered_copy_decline.c.
+//
+// - Two tasks reading a shared input, each mapping it `to` for itself, inside
+// a `target data` that already maps it. Each copies the input in before its
+// own kernel, and the other's copy writes the same bytes.
+// - Two kernels reaching data a `target data` has mapped through a pointer,
+// writing disjoint halves. Nothing in the graph copies that data.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 4096
+#define ITERS 4
+
+int main(void) {
+ int W[N], Out1[N], Out2[N];
+ int *T = (int *)malloc(N * sizeof(int));
+ int Dep1, Dep2, Errors = 0;
+
+ for (int I = 0; I < N; ++I)
+ W[I] = I;
+#pragma omp target data map(to : W[0 : N])
+ for (int It = 0; It < ITERS; ++It) {
+#pragma omp taskgraph
+ {
+#pragma omp target map(to : W[0 : N]) map(from : Out1[0 : N]) \
+ nowait depend(out : Dep1)
+ for (int I = 0; I < N; ++I)
+ Out1[I] = W[I] + It;
+#pragma omp target map(to : W[0 : N]) map(from : Out2[0 : N]) \
+ nowait depend(out : Dep2)
+ for (int I = 0; I < N; ++I)
+ Out2[I] = 2 * W[I] + It;
+ }
+#pragma omp taskwait
+ for (int I = 0; I < N; ++I)
+ Errors += Out1[I] != I + It || Out2[I] != 2 * I + It;
+ }
+
+ for (int I = 0; I < N; ++I)
+ T[I] = 0;
+#pragma omp target data map(tofrom : T[0 : N])
+ {
+ for (int It = 0; It < ITERS; ++It) {
+#pragma omp taskgraph
+ {
+#pragma omp target nowait depend(out : Dep1)
+ for (int I = 0; I < N / 2; ++I)
+ T[I] += 1;
+#pragma omp target nowait depend(out : Dep2)
+ for (int I = N / 2; I < N; ++I)
+ T[I] += 2;
+ }
+#pragma omp taskwait
+ }
+ }
+ for (int I = 0; I < N; ++I)
+ Errors += T[I] != (I < N / 2 ? ITERS : 2 * ITERS);
+
+ // CHECK: taskgraph unordered shared errors: 0
+ printf("taskgraph unordered shared errors: %d\n", Errors);
+ free(T);
+ return Errors != 0;
+}
>From c0dcb55cabeedf3d1a42198d3c84150a57019a39 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Mon, 19 Oct 2026 11:52:46 -0500
Subject: [PATCH 21/29] [Offload] Dump a taskgraph's data plan before each
replay
This patch adds debug-dumping for the taskgraph "data plan"
representation, under control of a new boolean-ish environment variable,
LIBOMPTARGET_TASKGRAPH_TRACE.
See new tests for example output, though they are XFAILed until the
first offload plugin lands.
Assisted-By: Claude
---
offload/include/Shared/TaskGraph.h | 4 +
offload/include/Shared/TaskGraphDebugDump.h | 232 ++++++++++++++++++
offload/libomptarget/interface.cpp | 15 ++
.../offloading/taskgraph_plan_dump_attach.c | 112 +++++++++
.../offloading/taskgraph_plan_dump_borrowed.c | 110 +++++++++
.../taskgraph_plan_dump_enter_exit.c | 148 +++++++++++
.../taskgraph_plan_dump_firstprivate.c | 123 ++++++++++
.../taskgraph_plan_dump_irreducible.c | 147 +++++++++++
8 files changed, 891 insertions(+)
create mode 100644 offload/include/Shared/TaskGraphDebugDump.h
create mode 100644 offload/test/offloading/taskgraph_plan_dump_attach.c
create mode 100644 offload/test/offloading/taskgraph_plan_dump_borrowed.c
create mode 100644 offload/test/offloading/taskgraph_plan_dump_enter_exit.c
create mode 100644 offload/test/offloading/taskgraph_plan_dump_firstprivate.c
create mode 100644 offload/test/offloading/taskgraph_plan_dump_irreducible.c
diff --git a/offload/include/Shared/TaskGraph.h b/offload/include/Shared/TaskGraph.h
index ef4bd206670982..ffc4697ab01b93 100644
--- a/offload/include/Shared/TaskGraph.h
+++ b/offload/include/Shared/TaskGraph.h
@@ -193,6 +193,7 @@ class TaskGraphArgBaseTy {
private:
friend class TaskGraphPlanner;
friend class TaskGraphDataPlanTy;
+ friend class TaskGraphDebugDump;
KindTy Kind = KindTy::None;
};
@@ -745,6 +746,7 @@ class TaskGraphDataPlanTy {
private:
friend class TaskGraphPlanner;
+ friend class TaskGraphDebugDump;
const char *attachStagingFor(const TaskGraphAttachArgTy &A) const {
return AttachStaging.data() + A.StagingOffset;
@@ -903,6 +905,7 @@ TaskGraphDataPlanTy::paramValue(const TaskGraphTargetNode &Leaf,
//===----------------------------------------------------------------------===//
class TaskGraphBuilder;
+class TaskGraphDebugDump;
/// Where the build pass links the next streamed element.
struct BuildCursor {
@@ -953,6 +956,7 @@ class TaskGraphTy {
private:
friend class TaskGraphBuilder;
+ friend class TaskGraphDebugDump;
TaskGraphElement *Root = nullptr;
std::vector<TaskGraphNode *> Leaves;
diff --git a/offload/include/Shared/TaskGraphDebugDump.h b/offload/include/Shared/TaskGraphDebugDump.h
new file mode 100644
index 00000000000000..7b4a5bdea08e38
--- /dev/null
+++ b/offload/include/Shared/TaskGraphDebugDump.h
@@ -0,0 +1,232 @@
+//===-- Shared/TaskGraphDebugDump.h - dump graph and data plan ---- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_SHARED_TASKGRAPHDEBUGDUMP_H
+#define OMPTARGET_SHARED_TASKGRAPHDEBUGDUMP_H
+
+#include "TaskGraph.h"
+#include "llvm/Support/Debug.h"
+
+namespace llvm {
+namespace omp {
+namespace target {
+
+class TaskGraphDebugDump {
+ TaskGraphTy &G;
+ llvm::raw_ostream &OS;
+
+ void dumpRanges(const llvm::ArrayRef<TaskGraphRangeTy> Ranges) {
+ for (const auto &R : Ranges) {
+ OS << "Id: " << R.getId() << ", ";
+ OS << "Host addr: " << R.getHostBegin() << ", ";
+ OS << "Device ptr: " << R.getDevicePtr() << ", ";
+ OS << "Size: " << R.getSize() << "\n";
+ }
+ }
+
+ void dumpCopy(const char *Dir, const TaskGraphCopyTy &Copy) {
+ if (Copy.Requested) {
+ OS << ", " << Dir << " (";
+ OS << (Copy.Perimeter ? "perimeter" : "internal");
+ OS << (Copy.Enabled ? ", enabled" : ", disabled");
+ OS << ")";
+ }
+ }
+
+ void dumpArg(const TaskGraphDataPlanTy &Plan, size_t I,
+ const TaskGraphArgSlotTy &Slot, int Indent = 0) {
+ const auto *Arg = &Slot.get();
+ // All the args are stored in a single array, and a particular construct
+ // uses a slice of that. The numbering shown is global/local. Updates
+ // use the global index; ops use the local index.
+ OS.indent(Indent) << Plan.argIndex(Slot) << "/" << I << ": ";
+ switch (Arg->getKind()) {
+ case TaskGraphArgBaseTy::KindTy::None:
+ OS << "none\n";
+ break;
+ case TaskGraphArgBaseTy::KindTy::Literal: {
+ const auto *L = cast<TaskGraphLiteralArgTy>(Arg);
+ OS << "literal (" << L->LiteralValue << ")\n";
+ break;
+ }
+ case TaskGraphArgBaseTy::KindTy::Mapped: {
+ const auto *M = cast<TaskGraphMappedArgTy>(Arg);
+ OS << "mapped (range: " << M->Range->getId() << ", offset: " << M->Offset
+ << ", size: " << M->Size;
+ dumpCopy("in", M->In);
+ dumpCopy("out", M->Out);
+ if (M->OwnsBorrowedOut)
+ OS << ", owns-borrowed-out";
+ if (M->Always)
+ OS << ", always";
+ OS << ")\n";
+ break;
+ }
+ case TaskGraphArgBaseTy::KindTy::Attach: {
+ const auto *A = cast<TaskGraphAttachArgTy>(Arg);
+ OS << "attach (ptr-range: " << A->PtrRange->getId()
+ << ", offset: " << A->Offset << ", size: " << A->Size
+ << ", ptee-range: " << A->PteeRange->getId()
+ << ", ptee-offset: " << A->PteeOffset
+ << ", staging-offset: " << A->StagingOffset;
+ if (A->Enabled)
+ OS << ", enabled)\n";
+ else
+ OS << ", disabled)\n";
+ break;
+ }
+ case TaskGraphArgBaseTy::KindTy::NoTargetAttach:
+ OS << "no-target-attach\n";
+ break;
+ case TaskGraphArgBaseTy::KindTy::FirstPrivate: {
+ const auto *F = cast<TaskGraphFirstPrivateArgTy>(Arg);
+ OS << "firstprivate (base-offset: " << F->BaseOffset
+ << ", size: " << F->Size;
+ if (F->Staged)
+ OS << ", staging-offset: " << F->StagingOffset;
+ else
+ OS << ", not-staged";
+ if (F->Saved)
+ OS << ", saved";
+ if (F->InitPending)
+ OS << ", init-pending";
+ OS << ")\n";
+ break;
+ }
+ }
+ }
+
+ void dumpOp(size_t I, const TaskGraphOpTy &Op, int Indent = 0) {
+ OS.indent(Indent) << I << ": ";
+ switch (Op.Kind) {
+ case TaskGraphOpTy::KindTy::Attach:
+ OS << "attach";
+ break;
+ case TaskGraphOpTy::KindTy::PrivateInit:
+ OS << "private-init";
+ break;
+ case TaskGraphOpTy::KindTy::CopyIn:
+ OS << "copy-in";
+ break;
+ case TaskGraphOpTy::KindTy::CopyOut:
+ OS << "copy-out";
+ break;
+ case TaskGraphOpTy::KindTy::Launch:
+ OS << "launch\n";
+ break;
+ }
+ if (Op.isCopyLike())
+ OS << " (arg: " << Op.Arg << ")\n";
+ }
+
+ void dumpLeaf(const TaskGraphDataPlanTy &Plan, const TaskGraphElement *Elem,
+ const TaskGraphRegion *Parent = nullptr, int Indent = 0) {
+ auto IrredSuccessors = [&](const TaskGraphRegion *Parent, int Indent) {
+ if (const auto *Irr =
+ dyn_cast_if_present<TaskGraphIrreducibleRegion>(Parent)) {
+ OS.indent(Indent) << "successors {\n";
+ const char *Sep = "";
+ for (size_t Succ = 0; Succ < Irr->NumEdges; ++Succ) {
+ const auto &Edge = Irr->edges()[Succ];
+ if (Edge.Src == (int32_t)Elem->ChildIndex) {
+ if (Sep[0] == '\0')
+ OS.indent(Indent + 2) << "-> ";
+ OS << Sep << "[" << Edge.Dst << "]";
+ Sep = ", ";
+ }
+ }
+ OS << "\n";
+ OS.indent(Indent) << "}\n";
+ }
+ };
+ if (isa_and_present<TaskGraphIrreducibleRegion>(Parent))
+ OS.indent(Indent) << "[" << Elem->ChildIndex << "] ";
+ else
+ OS << indent(Indent);
+ if (const auto *R = dyn_cast<const TaskGraphRegion>(Elem)) {
+ if (isa<const TaskGraphParRegion>(R)) {
+ OS << "parallel {\n";
+ } else if (isa<const TaskGraphSeqRegion>(R)) {
+ OS << "sequential {\n";
+ } else if (isa<const TaskGraphExclRegion>(R)) {
+ OS << "exclusive {\n";
+ } else if (isa<const TaskGraphIrreducibleRegion>(R)) {
+ OS << "irreducible {\n";
+ }
+ for (const auto *C : R->children()) {
+ dumpLeaf(Plan, C, R, Indent + 2);
+ }
+ IrredSuccessors(Parent, Indent + 2);
+ OS.indent(Indent) << "}\n";
+ } else if (const auto *L = dyn_cast<const TaskGraphNode>(Elem)) {
+ if (isa<const TaskGraphTargetNode>(L)) {
+ OS << "target {\n";
+ } else if (isa<const TaskGraphTargetEnterDataNode>(L)) {
+ OS << "enter data {\n";
+ } else if (isa<const TaskGraphTargetExitDataNode>(L)) {
+ OS << "exit data {\n";
+ } else if (isa<const TaskGraphTargetUpdateDataNode>(L)) {
+ OS << "update {\n";
+ }
+ OS.indent(Indent + 2) << "args {\n";
+ for (size_t I = 0; I < L->argPlan().size(); ++I)
+ dumpArg(Plan, I, L->argPlan()[I], Indent + 4);
+ OS.indent(Indent + 2) << "}\n";
+ OS.indent(Indent + 2) << "ops {\n";
+ for (size_t I = 0; I < L->ops().size(); ++I)
+ dumpOp(I, L->ops()[I], Indent + 4);
+ OS.indent(Indent + 2) << "}\n";
+ IrredSuccessors(Parent, Indent + 2);
+ OS.indent(Indent) << "}\n";
+ }
+ }
+
+public:
+ explicit TaskGraphDebugDump(TaskGraphTy &G, llvm::raw_ostream &OS)
+ : G(G), OS(OS) {}
+
+ void dumpDataPlan() {
+ OS << "Ranges:\n";
+ dumpRanges(G.Plan.ranges());
+ OS << "\n";
+ OS << "Firstprivate staging buffer size: " << G.Plan.privateStaging().size()
+ << ", device ptr: " << G.Plan.getPrivateBlock() << "\n\n";
+ OS << "attach staging buffer size: " << G.Plan.attachStaging().size()
+ << "\n\n";
+ OS << "Graph " << (void *)&G << ":\n";
+ dumpLeaf(G.Plan, G.getRoot());
+ }
+
+ void dumpUpdates(llvm::ArrayRef<TaskGraphArgUpdateTy> Updates, bool Resync) {
+ if (Resync) {
+ OS << "Updates all (resync)\n";
+ return;
+ }
+ OS << "Updates:\n";
+ for (const auto &U : Updates) {
+ OS << "Arg " << U.Arg << " changes:";
+ if (U.has(TaskGraphArgUpdateTy::HostAddressChanged))
+ OS << " host-addr";
+ if (U.has(TaskGraphArgUpdateTy::DeviceAddressChanged))
+ OS << " device-addr";
+ if (U.has(TaskGraphArgUpdateTy::DataChanged))
+ OS << " data";
+ if (U.has(TaskGraphArgUpdateTy::EnabledChanged))
+ OS << " enable";
+ OS << "\n";
+ }
+ if (Updates.empty())
+ OS << "(no updates)\n";
+ }
+};
+
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OMPTARGET_SHARED_TASKGRAPHDEBUGDUMP_H
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index 57ebd332aea580..78aa5207a40a85 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -27,6 +27,7 @@
#include "Shared/Profile.h"
#include "Shared/TaskGraph.h"
#include "Shared/TaskGraphBuilder.h"
+#include "Shared/TaskGraphDebugDump.h"
#include "Utils/ExponentialBackoff.h"
@@ -704,6 +705,7 @@ using llvm::omp::target::commitTaskGraphUpdates;
using llvm::omp::target::releaseTaskGraphPlan;
using llvm::omp::target::TaskGraphArgUpdateTy;
using llvm::omp::target::TaskGraphBuilder;
+using llvm::omp::target::TaskGraphDebugDump;
using llvm::omp::target::TaskGraphEdge;
using llvm::omp::target::TaskGraphElement;
using llvm::omp::target::TaskGraphExclRegion;
@@ -963,6 +965,11 @@ EXTERN void __tgt_taskgraph_begin_replay(void *Graph) {
beginTaskGraphReplay(*G);
}
+static bool dumpTaskGraphPlans() {
+ static BoolEnvar Dump("LIBOMPTARGET_TASKGRAPH_TRACE", false);
+ return Dump;
+}
+
EXTERN int __tgt_taskgraph_replay(void *Graph) {
auto *G = static_cast<TaskGraphTy *>(Graph);
assert(G->PluginGraph && "replaying a graph no plugin claimed");
@@ -993,6 +1000,14 @@ EXTERN int __tgt_taskgraph_replay(void *Graph) {
? nullptr
: "it cannot be replayed against the current mapping "
"state");
+
+ if (dumpTaskGraphPlans()) {
+ llvm::errs() << "Target graph pre-replay:\n";
+ TaskGraphDebugDump TGD(*G, llvm::errs());
+ TGD.dumpDataPlan();
+ TGD.dumpUpdates(Updates, Resync);
+ }
+
// Pinned every replay, since the ranges' host memory is only known to be the
// program's while the graph runs. A failed pin holds nothing.
if (Device.RTL->pin_taskgraph(Device.RTLDeviceID, G) != OFFLOAD_SUCCESS)
diff --git a/offload/test/offloading/taskgraph_plan_dump_attach.c b/offload/test/offloading/taskgraph_plan_dump_attach.c
new file mode 100644
index 00000000000000..992bdd76bb5801
--- /dev/null
+++ b/offload/test/offloading/taskgraph_plan_dump_attach.c
@@ -0,0 +1,112 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_TASKGRAPH_TRACE=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// The data plan of a graph mapping a section through a struct member pointer,
+// as dumped before each replay.
+//
+// The map attaches the device copy of the struct's pointer to the device copy
+// of the section. Each encounter of the graph points the struct at a different
+// live allocation, so every replay finds the section at a new host address.
+// The last replay finds the global array mapped outside the graph, which turns
+// its copy off.
+//
+// Range ids follow host addresses, so they are matched by size.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+typedef struct {
+ int *Q;
+} S;
+
+int K = 5;
+int Arr[100];
+
+static int run(int *Buf) {
+ struct {
+ int X, Y;
+ } Point = {7, 9};
+ S Mystruct = {Buf};
+
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : Mystruct.Q[4 : 50]) map(to : Arr[0 : 100]) \
+ firstprivate(Point)
+ for (int J = 4; J < 54; ++J)
+ Mystruct.Q[J] = Arr[J] + J + K + Point.X + Point.Y;
+ }
+
+ return Mystruct.Q[10];
+}
+
+int main(void) {
+ int *Bufs[3];
+ for (int I = 0; I < 3; ++I)
+ Bufs[I] = (int *)calloc(100, sizeof(int));
+ for (int I = 0; I < 100; ++I)
+ Arr[I] = I;
+
+ int Sum = run(Bufs[0]) + run(Bufs[1]);
+#pragma omp target enter data map(to : Arr[0 : 100])
+ Sum += run(Bufs[2]);
+#pragma omp target exit data map(release : Arr[0 : 100])
+
+ for (int I = 0; I < 3; ++I)
+ free(Bufs[I]);
+
+ // CHECK-LABEL: Target graph pre-replay:
+ // CHECK: Ranges:
+ // CHECK-DAG: Id: [[ARR:[0-9]+]], {{.*}}, Size: 400{{$}}
+ // CHECK-DAG: Id: [[Q:[0-9]+]], {{.*}}, Size: 200{{$}}
+ // CHECK-DAG: Id: [[S:[0-9]+]], {{.*}}, Size: 8{{$}}
+ // CHECK: Firstprivate staging buffer size: 8
+ // CHECK: attach staging buffer size: 8
+ // CHECK: target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 0/0: mapped (range: [[S]], offset: 0, size: 8, in (perimeter, enabled), out (perimeter, enabled))
+ // CHECK-NEXT: 1/1: mapped (range: [[Q]], offset: 0, size: 200, in (perimeter, enabled), out (perimeter, enabled))
+ // CHECK-NEXT: 2/2: attach (ptr-range: [[S]], offset: 0, size: 8, ptee-range: [[Q]], ptee-offset: 0, staging-offset: 0, enabled)
+ // CHECK-NEXT: 3/3: mapped (range: [[ARR]], offset: 0, size: 400, in (perimeter, enabled))
+ // CHECK-NEXT: 4/4: literal (5)
+ // CHECK-NEXT: 5/5: firstprivate (base-offset: 0, size: 8, staging-offset: 0, init-pending)
+ // CHECK-NEXT: 6/6: literal (0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: copy-in (arg: 0)
+ // CHECK-NEXT: 1: copy-in (arg: 1)
+ // CHECK-NEXT: 2: copy-in (arg: 3)
+ // CHECK-NEXT: 3: attach (arg: 2)
+ // CHECK-NEXT: 4: private-init (arg: 5)
+ // CHECK-NEXT: 5: launch
+ // CHECK-NEXT: 6: copy-out (arg: 0)
+ // CHECK-NEXT: 7: copy-out (arg: 1)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: Updates:
+ // CHECK-NEXT: Arg 0 changes: device-addr
+ // CHECK-NEXT: Arg 1 changes: host-addr device-addr
+ // CHECK-NEXT: Arg 2 changes: device-addr data enable
+ // CHECK-NEXT: Arg 3 changes: device-addr
+ // CHECK-NEXT: Arg 5 changes: data
+ //
+ // The section moved again, and the array is now borrowed.
+ // CHECK-LABEL: Target graph pre-replay:
+ // CHECK: 3/3: mapped (range: [[ARR]], offset: 0, size: 400, in (perimeter, disabled))
+ // CHECK: Updates:
+ // CHECK-NEXT: Arg 1 changes: host-addr
+ // CHECK-NEXT: Arg 3 changes: enable
+ // CHECK-NOT: Target graph pre-replay:
+ //
+ // Each run computes 10 + 10 + 5 + 7 + 9.
+ // CHECK: Sum = 123
+ printf("Sum = %d\n", Sum);
+ return 0;
+}
diff --git a/offload/test/offloading/taskgraph_plan_dump_borrowed.c b/offload/test/offloading/taskgraph_plan_dump_borrowed.c
new file mode 100644
index 00000000000000..7545c5e340add9
--- /dev/null
+++ b/offload/test/offloading/taskgraph_plan_dump_borrowed.c
@@ -0,0 +1,110 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_TASKGRAPH_TRACE=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// The data plan of a graph whose leaves exercise the remaining argument
+// properties, as dumped before each replay.
+//
+// - A section through a plain pointer: the pointer itself is not mapped, so
+// its attachment has no target.
+// - `always`, which copies whoever created the mapping.
+// - `target update`, which transfers whoever mapped the range.
+// - An exit data dropping the last reference to a mapping made outside the
+// graph, which therefore performs the copy out.
+//
+// Range ids follow host addresses, so they are matched by size.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int A[64], B[32];
+
+int main(void) {
+ int *P = (int *)calloc(48, sizeof(int));
+ for (int It = 0; It < 3; ++It) {
+#pragma omp target enter data map(alloc : B[0 : 32])
+ B[0] = It;
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : P[0 : 48])
+ P[0] += 1;
+#pragma omp target map(always, tofrom : A[0 : 64])
+ A[0] += 1;
+#pragma omp target update to(B[0 : 32])
+#pragma omp target map(tofrom : B[0 : 32])
+ B[0] += 10;
+#pragma omp target exit data map(from : B[0 : 32])
+ }
+ }
+
+ // CHECK-LABEL: Target graph pre-replay:
+ // CHECK: Ranges:
+ // CHECK-DAG: Id: [[A:[0-9]+]], {{.*}}, Size: 256{{$}}
+ // CHECK-DAG: Id: [[B:[0-9]+]], {{.*}}, Size: 128{{$}}
+ // CHECK-DAG: Id: [[P:[0-9]+]], {{.*}}, Size: 192{{$}}
+ // CHECK: Firstprivate staging buffer size: 0
+ // CHECK: sequential {
+ // CHECK-NEXT: target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 0/0: mapped (range: [[P]], offset: 0, size: 192, in (perimeter, enabled), out (perimeter, enabled))
+ // CHECK-NEXT: 1/1: no-target-attach
+ // CHECK-NEXT: 2/2: literal (0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: copy-in (arg: 0)
+ // CHECK-NEXT: 1: launch
+ // CHECK-NEXT: 2: copy-out (arg: 0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 3/0: mapped (range: [[A]], offset: 0, size: 256, in (perimeter, enabled), out (perimeter, enabled), always)
+ // CHECK-NEXT: 4/1: literal (0)
+ // CHECK-NEXT: }
+ // CHECK: update {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 5/0: mapped (range: [[B]], offset: 0, size: 128, in (internal, enabled), always)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: copy-in (arg: 0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 6/0: mapped (range: [[B]], offset: 0, size: 128, in (perimeter, disabled), out (perimeter, disabled))
+ // CHECK-NEXT: 7/1: literal (0)
+ // CHECK-NEXT: }
+ // CHECK: exit data {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 8/0: mapped (range: [[B]], offset: 0, size: 128, out (internal, enabled), owns-borrowed-out)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: copy-out (arg: 0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: Updates:
+ // CHECK-NEXT: Arg 0 changes: device-addr
+ // CHECK-NEXT: Arg 3 changes: device-addr
+ // CHECK-NEXT: Arg 5 changes: device-addr
+ // CHECK-NEXT: Arg 6 changes: device-addr enable
+ // CHECK-NEXT: Arg 8 changes: device-addr
+ //
+ // The second replay finds everything as the first left it.
+ // CHECK-LABEL: Target graph pre-replay:
+ // CHECK: Updates:
+ // CHECK-NEXT: (no updates)
+ // CHECK-NOT: Target graph pre-replay:
+ //
+ // CHECK: P[0] = 3, A[0] = 3, B[0] = 12
+ printf("P[0] = %d, A[0] = %d, B[0] = %d\n", P[0], A[0], B[0]);
+ free(P);
+ return 0;
+}
diff --git a/offload/test/offloading/taskgraph_plan_dump_enter_exit.c b/offload/test/offloading/taskgraph_plan_dump_enter_exit.c
new file mode 100644
index 00000000000000..60798389fe16d1
--- /dev/null
+++ b/offload/test/offloading/taskgraph_plan_dump_enter_exit.c
@@ -0,0 +1,148 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_TASKGRAPH_TRACE=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// The data plan of a graph that maps its data with enter and exit data around a
+// target region, as dumped before each replay.
+//
+// The enter data creates the mappings, so its maps perform the copies in and
+// the exit data's perform the copies out; the target region's implicit maps
+// fall inside them and are internal, their copies off. Mapping the struct
+// together with the section lets the enter data attach its pointer. The last
+// replay finds the global array mapped outside the graph, which turns its
+// copies off.
+//
+// Range ids follow host addresses, so they are matched by size.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+typedef struct {
+ int *Q;
+} S;
+
+int K = 5;
+int Arr[100];
+
+static int run(int *Buf) {
+ struct {
+ int X, Y;
+ } Point = {7, 9};
+ S Mystruct = {Buf};
+
+#pragma omp taskgraph
+ {
+#pragma omp target enter data map(to : Mystruct, Mystruct.Q[0 : 50], \
+ Arr[0 : 100])
+#pragma omp target firstprivate(Point)
+ for (int J = 4; J < 50; ++J)
+ Mystruct.Q[J] = Arr[J] + J + K + Point.X + Point.Y;
+#pragma omp target exit data map(from : Mystruct, Mystruct.Q[0 : 50], \
+ Arr[0 : 100])
+ }
+
+ return Mystruct.Q[10];
+}
+
+int main(void) {
+ int *Bufs[3];
+ for (int I = 0; I < 3; ++I)
+ Bufs[I] = (int *)calloc(100, sizeof(int));
+ for (int I = 0; I < 100; ++I)
+ Arr[I] = I;
+
+ int Sum = run(Bufs[0]) + run(Bufs[1]);
+#pragma omp target enter data map(to : Arr[0 : 100])
+ Sum += run(Bufs[2]);
+#pragma omp target exit data map(release : Arr[0 : 100])
+
+ for (int I = 0; I < 3; ++I)
+ free(Bufs[I]);
+
+ // CHECK-LABEL: Target graph pre-replay:
+ // CHECK: Ranges:
+ // CHECK-DAG: Id: [[ARR:[0-9]+]], {{.*}}, Size: 400{{$}}
+ // CHECK-DAG: Id: [[Q:[0-9]+]], {{.*}}, Size: 200{{$}}
+ // CHECK-DAG: Id: [[S:[0-9]+]], {{.*}}, Size: 8{{$}}
+ // CHECK: Firstprivate staging buffer size: 8
+ // CHECK: attach staging buffer size: 8
+ // CHECK: sequential {
+ // CHECK-NEXT: enter data {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 0/0: mapped (range: [[S]], offset: 0, size: 8, in (perimeter, enabled))
+ // CHECK-NEXT: 1/1: mapped (range: [[Q]], offset: 0, size: 200, in (perimeter, enabled))
+ // CHECK-NEXT: 2/2: attach (ptr-range: [[S]], offset: 0, size: 8, ptee-range: [[Q]], ptee-offset: 0, staging-offset: 0, enabled)
+ // CHECK-NEXT: 3/3: mapped (range: [[ARR]], offset: 0, size: 400, in (perimeter, enabled))
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: copy-in (arg: 0)
+ // CHECK-NEXT: 1: copy-in (arg: 1)
+ // CHECK-NEXT: 2: copy-in (arg: 3)
+ // CHECK-NEXT: 3: attach (arg: 2)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 4/0: mapped (range: [[S]], offset: 0, size: 8, in (internal, disabled), out (internal, disabled))
+ // CHECK-NEXT: 5/1: mapped (range: [[ARR]], offset: 0, size: 400, in (internal, disabled), out (internal, disabled))
+ // CHECK-NEXT: 6/2: literal (5)
+ // CHECK-NEXT: 7/3: firstprivate (base-offset: 0, size: 8, staging-offset: 0, init-pending)
+ // CHECK-NEXT: 8/4: literal (0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: copy-in (arg: 0)
+ // CHECK-NEXT: 1: copy-in (arg: 1)
+ // CHECK-NEXT: 2: private-init (arg: 3)
+ // CHECK-NEXT: 3: launch
+ // CHECK-NEXT: 4: copy-out (arg: 0)
+ // CHECK-NEXT: 5: copy-out (arg: 1)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: exit data {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 9/0: mapped (range: [[S]], offset: 0, size: 8, out (perimeter, enabled))
+ // CHECK-NEXT: 10/1: mapped (range: [[Q]], offset: 0, size: 200, out (perimeter, enabled))
+ // CHECK-NEXT: 11/2: mapped (range: [[ARR]], offset: 0, size: 400, out (perimeter, enabled))
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: copy-out (arg: 0)
+ // CHECK-NEXT: 1: copy-out (arg: 1)
+ // CHECK-NEXT: 2: copy-out (arg: 2)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: Updates:
+ // CHECK-NEXT: Arg 0 changes: device-addr
+ // CHECK-NEXT: Arg 1 changes: host-addr device-addr
+ // CHECK-NEXT: Arg 2 changes: device-addr data enable
+ // CHECK-NEXT: Arg 3 changes: device-addr
+ // CHECK-NEXT: Arg 4 changes: device-addr enable
+ // CHECK-NEXT: Arg 5 changes: device-addr enable
+ // CHECK-NEXT: Arg 7 changes: data
+ // CHECK-NEXT: Arg 9 changes: device-addr
+ // CHECK-NEXT: Arg 10 changes: host-addr device-addr
+ // CHECK-NEXT: Arg 11 changes: device-addr
+ //
+ // The section moved again, and the array is now borrowed.
+ // CHECK-LABEL: Target graph pre-replay:
+ // CHECK: 3/3: mapped (range: [[ARR]], offset: 0, size: 400, in (perimeter, disabled))
+ // CHECK: 11/2: mapped (range: [[ARR]], offset: 0, size: 400, out (perimeter, disabled))
+ // CHECK: Updates:
+ // CHECK-NEXT: Arg 1 changes: host-addr
+ // CHECK-NEXT: Arg 3 changes: enable
+ // CHECK-NEXT: Arg 10 changes: host-addr
+ // CHECK-NEXT: Arg 11 changes: enable
+ // CHECK-NOT: Target graph pre-replay:
+ //
+ // Each run computes 10 + 10 + 5 + 7 + 9.
+ // CHECK: Sum = 123
+ printf("Sum = %d\n", Sum);
+ return 0;
+}
diff --git a/offload/test/offloading/taskgraph_plan_dump_firstprivate.c b/offload/test/offloading/taskgraph_plan_dump_firstprivate.c
new file mode 100644
index 00000000000000..d8506506dc5c5b
--- /dev/null
+++ b/offload/test/offloading/taskgraph_plan_dump_firstprivate.c
@@ -0,0 +1,123 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_TASKGRAPH_TRACE=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// The data plan of a graph with privatised and by-value arguments, as dumped
+// before each replay.
+//
+// - An aggregate firstprivate without `saved` is re-staged every replay; one
+// with `saved` is staged once, from the value at recording.
+// - A scalar firstprivate is a literal, re-read every replay, so a change in
+// it is reported as a data update.
+// - Two sections of an array mapped outside the graph merge into one range,
+// and their copies are off, the outside mapping standing in for them.
+//
+// Range ids follow host addresses, so they are matched by size. The staged
+// aggregates are 16 bytes so that their staging offsets do not depend on the
+// alignment of their host addresses.
+
+#include <stdio.h>
+
+struct Quad {
+ int X, Y, Z, W;
+};
+
+int K = 5;
+
+int main(void) {
+ int Arr[100], Arr2[100];
+ struct Quad Q = {0, 2, 0, 0};
+ for (int I = 0; I < 100; ++I)
+ Arr2[I] = I;
+
+#pragma omp target enter data map(to : Arr2[0 : 100])
+ for (int It = 0; It < 3; ++It) {
+ int Scale = It;
+ Q.X = It;
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : Arr[0 : 100]) map(to : Arr2[4 : 50]) \
+ firstprivate(saved : K) firstprivate(Q, Scale)
+ for (int J = 0; J < 100; ++J)
+ Arr[J] = J * Scale + K + Q.X + Q.Y;
+#pragma omp target map(tofrom : Arr[0 : 100]) map(from : Arr2[40 : 50]) \
+ firstprivate(saved : K, Q)
+ for (int J = 0; J < 100; ++J)
+ Arr[J] += J + K + Q.X + Q.Y;
+ }
+ }
+#pragma omp target exit data map(from : Arr2[0 : 100])
+
+ // CHECK-LABEL: Target graph pre-replay:
+ // CHECK: Ranges:
+ // CHECK-DAG: Id: [[ARR2:[0-9]+]], {{.*}}, Size: 344{{$}}
+ // CHECK-DAG: Id: [[ARR:[0-9]+]], {{.*}}, Size: 400{{$}}
+ // CHECK: Firstprivate staging buffer size: 32
+ // CHECK: attach staging buffer size: 0
+ // CHECK: sequential {
+ // CHECK-NEXT: target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 0/0: mapped (range: [[ARR]], offset: 0, size: 400, in (perimeter, enabled), out (perimeter, enabled))
+ // CHECK-NEXT: 1/1: literal (1)
+ // CHECK-NEXT: 2/2: literal (5)
+ // CHECK-NEXT: 3/3: firstprivate (base-offset: 0, size: 16, staging-offset: 0, init-pending)
+ // CHECK-NEXT: 4/4: mapped (range: [[ARR2]], offset: 0, size: 200, in (perimeter, disabled))
+ // CHECK-NEXT: 5/5: literal (0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: copy-in (arg: 0)
+ // CHECK-NEXT: 1: copy-in (arg: 4)
+ // CHECK-NEXT: 2: private-init (arg: 3)
+ // CHECK-NEXT: 3: launch
+ // CHECK-NEXT: 4: copy-out (arg: 0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 6/0: mapped (range: [[ARR]], offset: 0, size: 400, in (perimeter, enabled), out (perimeter, enabled))
+ // CHECK-NEXT: 7/1: literal (5)
+ // CHECK-NEXT: 8/2: firstprivate (base-offset: 0, size: 16, staging-offset: 16, saved, init-pending)
+ // CHECK-NEXT: 9/3: mapped (range: [[ARR2]], offset: 144, size: 200, out (perimeter, disabled))
+ // CHECK-NEXT: 10/4: literal (0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: copy-in (arg: 0)
+ // CHECK-NEXT: 1: private-init (arg: 2)
+ // CHECK-NEXT: 2: launch
+ // CHECK-NEXT: 3: copy-out (arg: 0)
+ // CHECK-NEXT: 4: copy-out (arg: 3)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: Updates:
+ // CHECK-NEXT: Arg 0 changes: device-addr
+ // CHECK-NEXT: Arg 1 changes: data
+ // CHECK-NEXT: Arg 3 changes: data
+ // CHECK-NEXT: Arg 4 changes: device-addr enable
+ // CHECK-NEXT: Arg 6 changes: device-addr
+ // CHECK-NEXT: Arg 8 changes: data
+ // CHECK-NEXT: Arg 9 changes: device-addr enable
+ //
+ // The second replay re-reads the scalar and re-stages the unsaved aggregate,
+ // and nothing else changes.
+ // CHECK-LABEL: Target graph pre-replay:
+ // CHECK: 1/1: literal (2)
+ // CHECK: 3/3: firstprivate (base-offset: 0, size: 16, staging-offset: 0, init-pending)
+ // CHECK: 8/2: firstprivate (base-offset: 0, size: 16, staging-offset: 16, saved)
+ // CHECK: Updates:
+ // CHECK-NEXT: Arg 1 changes: data
+ // CHECK-NEXT: Arg 3 changes: data
+ // CHECK-NOT: Target graph pre-replay:
+ //
+ // The last replay: 1 * 2 + 5 + 2 + 2, then + 1 + 5 + 0 + 2 from the saved Q.
+ // CHECK: Arr[1] = 19
+ printf("Arr[1] = %d\n", Arr[1]);
+ return 0;
+}
diff --git a/offload/test/offloading/taskgraph_plan_dump_irreducible.c b/offload/test/offloading/taskgraph_plan_dump_irreducible.c
new file mode 100644
index 00000000000000..c7268298af4786
--- /dev/null
+++ b/offload/test/offloading/taskgraph_plan_dump_irreducible.c
@@ -0,0 +1,147 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_TASKGRAPH_TRACE=1 \
+// RUN: %libomptarget-run-generic 2>&1 | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// The data plan of a graph whose dependences are not series-parallel, as dumped
+// before each replay.
+//
+// TA TB
+// | \ |
+// | \ |
+// TC '-- TD
+// |
+// TE
+//
+// TA, TB, TC and TD form an N, the smallest shape with no series-parallel
+// decomposition, so they land in an irreducible region whose edges are
+// transmitted explicitly. TE only follows TC, so the two of them are carved
+// into a sequential region, which is one of the irreducible region's children.
+// Each child lists the children it is a predecessor of, a leaf and a region
+// alike.
+//
+// `nowait` keeps the depend clauses the only edges: an undeferred target is
+// ordered after everything encountered before it, which would make the graph a
+// total order.
+//
+// Each leaf maps an array of its own size, so its range is identified by size.
+
+#include <stdio.h>
+
+int A[1], B[2], C[3], D[4], E[5];
+
+int main(void) {
+ int Da, Db, Dc;
+ for (int It = 0; It < 2; ++It) {
+#pragma omp taskgraph
+ {
+#pragma omp target nowait map(from : A[0 : 1]) depend(out : Da)
+ A[0] = 1;
+#pragma omp target nowait map(from : B[0 : 2]) depend(out : Db)
+ B[0] = 2;
+#pragma omp target nowait map(from : C[0 : 3]) depend(in : Da) depend(out : Dc)
+ C[0] = 3;
+#pragma omp target nowait map(from : D[0 : 4]) depend(in : Da, Db)
+ D[0] = 4;
+#pragma omp target nowait map(from : E[0 : 5]) depend(in : Dc)
+ E[0] = 5;
+ }
+ }
+
+ // The children are numbered in the order libomp carves them, which puts the
+ // sequential region first.
+ //
+ // CHECK-LABEL: Target graph pre-replay:
+ // CHECK: Ranges:
+ // CHECK-DAG: Id: [[A:[0-9]+]], {{.*}}, Size: 4{{$}}
+ // CHECK-DAG: Id: [[B:[0-9]+]], {{.*}}, Size: 8{{$}}
+ // CHECK-DAG: Id: [[C:[0-9]+]], {{.*}}, Size: 12{{$}}
+ // CHECK-DAG: Id: [[D:[0-9]+]], {{.*}}, Size: 16{{$}}
+ // CHECK-DAG: Id: [[E:[0-9]+]], {{.*}}, Size: 20{{$}}
+ // CHECK: Firstprivate staging buffer size: 0
+ // CHECK: attach staging buffer size: 0
+ // CHECK: irreducible {
+ // CHECK-NEXT: [0] sequential {
+ // CHECK-NEXT: target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 0/0: mapped (range: [[C]], offset: 0, size: 12, out (perimeter, enabled))
+ // CHECK-NEXT: 1/1: literal (0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: launch
+ // CHECK-NEXT: 1: copy-out (arg: 0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 2/0: mapped (range: [[E]], offset: 0, size: 20, out (perimeter, enabled))
+ // CHECK-NEXT: 3/1: literal (0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: launch
+ // CHECK-NEXT: 1: copy-out (arg: 0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: successors {
+ // CHECK-EMPTY:
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: [1] target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 4/0: mapped (range: [[A]], offset: 0, size: 4, out (perimeter, enabled))
+ // CHECK-NEXT: 5/1: literal (0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: launch
+ // CHECK-NEXT: 1: copy-out (arg: 0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: successors {
+ // CHECK-NEXT: -> [0], [2]{{$}}
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: [2] target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 6/0: mapped (range: [[D]], offset: 0, size: 16, out (perimeter, enabled))
+ // CHECK-NEXT: 7/1: literal (0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: launch
+ // CHECK-NEXT: 1: copy-out (arg: 0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: successors {
+ // CHECK-EMPTY:
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: [3] target {
+ // CHECK-NEXT: args {
+ // CHECK-NEXT: 8/0: mapped (range: [[B]], offset: 0, size: 8, out (perimeter, enabled))
+ // CHECK-NEXT: 9/1: literal (0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: ops {
+ // CHECK-NEXT: 0: launch
+ // CHECK-NEXT: 1: copy-out (arg: 0)
+ // CHECK-NEXT: }
+ // CHECK-NEXT: successors {
+ // CHECK-NEXT: -> [2]{{$}}
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: }
+ // CHECK-NEXT: Updates:
+ // CHECK-NEXT: Arg 0 changes: device-addr
+ // CHECK-NEXT: Arg 2 changes: device-addr
+ // CHECK-NEXT: Arg 4 changes: device-addr
+ // CHECK-NEXT: Arg 6 changes: device-addr
+ // CHECK-NEXT: Arg 8 changes: device-addr
+ // CHECK-NOT: Target graph pre-replay:
+ //
+ // CHECK: A[0] + B[0] + C[0] + D[0] + E[0] = 15
+ printf("A[0] + B[0] + C[0] + D[0] + E[0] = %d\n",
+ A[0] + B[0] + C[0] + D[0] + E[0]);
+ return 0;
+}
>From 820b858e146db1200706c55f31fbc0be26751e1e Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 1 Oct 2026 06:43:55 -0500
Subject: [PATCH 22/29] [OpenMP][Offload] Runtime support for a 'target data'
block in a taskgraph
In OpenMP 6.0 (15.7) the semantics of a 'target data' block construct
are that it is task-generating, and the contents of the block are a
"sharing task". In the taskgraph context, that implies that the contents
of the block are *themselves* a task: thus, any host tasks or target
constructs *within* a "target data" construct will *not* be recorded as
part of the taskgraph (but the "sharing task" will be).
This patch implements the runtime parts of that: a new tasking flag to
recognize the body of a "target data" construct, and a helper function for
"use_device_ptr"/"use_device_addr" clauses (used by the next patch). The
"target data" block executes in three parts: an "enter data" operation,
the (host) task launch for the body, then an "exit data" operation.
Assisted-By: Claude
---
offload/include/omptarget.h | 13 ++++++++++
offload/libomptarget/exports | 1 +
offload/libomptarget/interface.cpp | 38 ++++++++++++++++++++++++++++++
openmp/runtime/src/kmp.h | 6 +++--
openmp/runtime/src/kmp_tasking.cpp | 14 +++++++++++
5 files changed, 70 insertions(+), 2 deletions(-)
diff --git a/offload/include/omptarget.h b/offload/include/omptarget.h
index e0cd2e8a046d5c..a4d6afa6b01fd3 100644
--- a/offload/include/omptarget.h
+++ b/offload/include/omptarget.h
@@ -634,6 +634,19 @@ void __tgt_taskgraph_emit_target_update(void *Graph, int64_t DeviceId,
int64_t *ArgTypes, void **ArgNames,
void **ArgMappers);
+/// Answer, for every `RETURN_PARAM` entry of \p ArgTypes, where the entry's
+/// memory currently is on \p DeviceId, writing the answers into \p Out.
+///
+/// This is the `use_device_ptr` / `use_device_addr` rule of `targetDataBegin`
+/// with none of the mapping: the device address is offset back by the entry's
+/// own `HstPtrBegin - HstPtrBase` delta so that the list item's base is what
+/// comes out, no reference count moves, and an absent entry falls back exactly
+/// as the mapper's does. A recorded `target data` region's body asks it on
+/// every replay, after the region's enter-data node has done the mapping.
+void __tgt_taskgraph_lookup_use_device(int64_t DeviceId, int32_t ArgNum,
+ void **ArgsBase, void **Args,
+ int64_t *ArgTypes, void **Out);
+
/// Finish transmitting a subgraph. Returns the number of bytes the subgraph
/// occupies (the single-block size): meaningful on the first (measuring) pass,
/// where its return value is fed back to __tgt_taskgraph_start as
diff --git a/offload/libomptarget/exports b/offload/libomptarget/exports
index 5e086c86b7a015..f1d6dc61907840 100644
--- a/offload/libomptarget/exports
+++ b/offload/libomptarget/exports
@@ -45,6 +45,7 @@ VERS1.0 {
__tgt_taskgraph_emit_target_enter_data;
__tgt_taskgraph_emit_target_exit_data;
__tgt_taskgraph_emit_target_update;
+ __tgt_taskgraph_lookup_use_device;
__tgt_taskgraph_end;
__tgt_taskgraph_finalize;
__tgt_taskgraph_begin_replay;
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index 78aa5207a40a85..4181959d15eb7c 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -848,6 +848,44 @@ EXTERN void __tgt_taskgraph_emit_target_update(
ArgMappers);
}
+/// Answer a recorded `target data` region's use_device_ptr / use_device_addr
+/// list items, as the begin mapper would, when the region's body starts on a
+/// replay. \p Out receives one device base per entry.
+EXTERN void __tgt_taskgraph_lookup_use_device(int64_t DeviceId, int32_t ArgNum,
+ void **ArgsBase, void **Args,
+ int64_t *ArgTypes, void **Out) {
+ assert(PM && "Runtime not initialized");
+ auto DeviceOrErr = PM->getDevice(resolveGraphDevice(DeviceId));
+ if (!DeviceOrErr)
+ FATAL_MESSAGE(DeviceId, "%s", toString(DeviceOrErr.takeError()).c_str());
+ MappingInfoTy &MappingInfo = DeviceOrErr->getMappingInfo();
+
+ for (int32_t I = 0; I < ArgNum; ++I) {
+ if (!(ArgTypes[I] & OMP_TGT_MAPTYPE_RETURN_PARAM))
+ continue;
+ const uintptr_t Delta = reinterpret_cast<uintptr_t>(Args[I]) -
+ reinterpret_cast<uintptr_t>(ArgsBase[I]);
+ // Deliberately no reference-count movement and no hold: this is the
+ // question `where is it`, asked of a mapping some map is already
+ // responsible for. A graph asks it once per replay, so answering it with
+ // a reference would leak one per replay.
+ TargetPointerResultTy TPR = MappingInfo.getTgtPtrBegin(
+ Args[I], /*Size=*/0, /*UpdateRefCount=*/false,
+ /*UseHoldRefCount=*/false);
+ uintptr_t Begin = reinterpret_cast<uintptr_t>(TPR.TargetPointer);
+ // An absent entry comes back as the host address, so it is told apart by
+ // the flags rather than by a null. It takes the same two fallbacks as in
+ // the mapper: nullify when the clause asked for it, and otherwise hand back
+ // the host address, which OpenMP makes the program's responsibility to
+ // have made device-accessible. Host-resident memory under unified shared
+ // memory is found, not absent.
+ if (!TPR.isPresent() && !TPR.isHostPointer() &&
+ (ArgTypes[I] & OMP_TGT_MAPTYPE_FB_NULLIFY))
+ Begin = 0;
+ Out[I] = reinterpret_cast<void *>(Begin - Delta);
+ }
+}
+
EXTERN size_t __tgt_taskgraph_end(void *Graph) {
auto *G = static_cast<TaskGraphTy *>(Graph);
TaskGraphBuilder B(*G);
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index b2be1c0033678f..488c08b71c1cc3 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2972,7 +2972,8 @@ typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
unsigned tasking_ser : 1;
unsigned task_serial : 1;
unsigned tasktype : 1;
- unsigned reserved : 6;
+ unsigned reserved : 5;
+ unsigned taskgraph_body : 1;
unsigned taskgraph_destructors : 1;
unsigned transparent : 1;
unsigned free_agent_eligible : 1;
@@ -3003,7 +3004,8 @@ typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
argument, which of the task's initial
values to destroy (a task that may be
recorded into a taskgraph) */
- unsigned reserved : 6; /* reserved for compiler use */
+ unsigned taskgraph_body : 1; /* structured block of a recorded target data */
+ unsigned reserved : 5; /* reserved for compiler use */
/* Library flags */ /* Total library flags must be 16 bits */
unsigned tasktype : 1; /* task is either explicit(1) or implicit (0) */
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index aa28427abf50e5..d6ca2c9a778fd1 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -6872,6 +6872,10 @@ kmp_uint32 __kmpc_taskgraph_task(ident_t *loc_ref, kmp_int32 gtid,
kmp_task_t *cloned_task = __kmp_taskgraph_clone_task(
thread, rec, new_task, sizeof_kmp_task_t, sizeof_shareds, task_dup);
kmp_taskgraph_node_t *node = __kmp_taskgraph_node_alloc(rec, cloned_task);
+ // The body of a 'target data' is ordered between its enter-data and
+ // exit-data nodes, which have no dependences on it to express that.
+ node->undeferred =
+ KMP_TASK_TO_TASKDATA(new_task)->td_flags.taskgraph_body;
if (taskgroup->taskgraph.reduce_input) {
node->reduce_input = taskgroup->taskgraph.reduce_input;
taskgroup->taskgraph.reduce_input = nullptr;
@@ -6911,6 +6915,16 @@ kmp_uint32 __kmpc_taskgraph_task(ident_t *loc_ref, kmp_int32 gtid,
}
}
+ // The body of a 'target data' finishes before the construct does: the
+ // exit-data action that follows it must see everything the body did.
+ if (KMP_TASK_TO_TASKDATA(new_task)->td_flags.taskgraph_body) {
+ KMP_DEBUG_ASSERT(ndeps == 0);
+ __kmpc_omp_task_begin_if0(loc_ref, gtid, new_task);
+ new_task->routine(gtid, new_task);
+ __kmpc_omp_task_complete_if0(loc_ref, gtid, new_task);
+ return 0;
+ }
+
kmp_int32 res;
if (ndeps == 0)
res = __kmpc_omp_task(loc_ref, gtid, new_task);
>From f2d0fd6e401343e389ae15b050e31be08307c3a2 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 2 Oct 2026 05:18:20 -0500
Subject: [PATCH 23/29] [Clang][OpenMP] Frontend parts for 'target data' block
in taskgraph
These are the front-end parts to treat the body of a 'target data'
block within a taskgraph as its own task.
Target addresses for "use_device_ptr" and "use_device_addr" are looked
up from the body of the generated task itself via a libomptarget helper,
__tgt_taskgraph_lookup_use_device. It might not be immediately apparent
why the existing relocation mechanism for tasks is not sufficient for this
case: the reason is that relocation is done at the start of a replay,
and an "enter data" operation might cause a new target address to be
allocated for the mapping partway through the replay.
Because it might not be obvious that tasks within 'target data' may not
be recorded directly in the taskgraph, a warning under -Wopenmp-taskgraph
informs the user about that. Dependencies listed on tasks within the
'target data' block also cannot interact with those of tasks that are
recorded in the taskgraph proper, so those are also diagnosed where
possible.
Tests have been added to cover new functionality, some of which are
XFAILed until a plugin claims the graph.
Assisted-By: Claude
Pull Request: https://github.com/llvm/llvm-project/pull/225722
---
clang/include/clang/Basic/DiagnosticGroups.td | 3 +-
.../clang/Basic/DiagnosticSemaKinds.td | 11 +
clang/lib/CodeGen/CGOpenMPRuntime.cpp | 216 +++++++++++++++---
clang/lib/CodeGen/CGOpenMPRuntime.h | 61 +++--
clang/lib/CodeGen/CGStmtOpenMP.cpp | 53 ++++-
clang/lib/Sema/SemaOpenMP.cpp | 175 +++++++++++++-
.../taskgraph_target_data_block_codegen.cpp | 155 +++++++++++++
.../taskgraph_target_data_body_messages.cpp | 157 +++++++++++++
.../test/Parser/OpenMP/replayable-clause.f90 | 26 +++
llvm/include/llvm/Frontend/OpenMP/OMP.td | 1 +
.../include/llvm/Frontend/OpenMP/OMPKinds.def | 2 +
.../test/offloading/taskgraph_map_overlap.c | 147 ++++++++++++
.../offloading/taskgraph_target_data_block.c | 85 +++++++
.../offloading/taskgraph_target_data_body.c | 98 ++++++++
.../taskgraph_target_data_body_visible.c | 100 ++++++++
.../offloading/taskgraph_use_device_ptr.c | 93 ++++++++
.../taskgraph_use_device_ptr_fallback.c | 63 +++++
17 files changed, 1394 insertions(+), 52 deletions(-)
create mode 100644 clang/test/OpenMP/taskgraph_target_data_block_codegen.cpp
create mode 100644 clang/test/OpenMP/taskgraph_target_data_body_messages.cpp
create mode 100644 offload/test/offloading/taskgraph_map_overlap.c
create mode 100644 offload/test/offloading/taskgraph_target_data_block.c
create mode 100644 offload/test/offloading/taskgraph_target_data_body.c
create mode 100644 offload/test/offloading/taskgraph_target_data_body_visible.c
create mode 100644 offload/test/offloading/taskgraph_use_device_ptr.c
create mode 100644 offload/test/offloading/taskgraph_use_device_ptr_fallback.c
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index b3bb2fb1b53fdc..b94e0685bb4142 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -1793,10 +1793,11 @@ def OpenMP51Ext : DiagGroup<"openmp-51-extensions">;
def OpenMPExtensions : DiagGroup<"openmp-extensions">;
def OpenMPTargetException : DiagGroup<"openmp-target-exception">;
def OpenMPFuture : DiagGroup<"openmp-future">;
+def OpenMPTaskgraph : DiagGroup<"openmp-taskgraph">;
def OpenMP : DiagGroup<"openmp", [
SourceUsesOpenMP, OpenMPClauses, OpenMPLoopForm, OpenMPTarget,
OpenMPMapping, OpenMP51Ext, OpenMPExtensions, OpenMPTargetException,
- OpenMPFuture
+ OpenMPFuture, OpenMPTaskgraph
]>;
// OpenACC warnings.
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 878c28cd256300..f16fd0a58e5dae 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12496,6 +12496,17 @@ def err_omp_taskgraph_replayable_task: Error<
"%select{detachable|transparent|undeferred}0 replayable task is not allowed within '#pragma omp taskgraph'">;
def err_omp_taskgraph_not_task_generating: Error<
"'%0' directive is not task-generating and cannot be used within '#pragma omp taskgraph'">;
+def warn_omp_taskgraph_body_device_construct : Warning<
+ "'#pragma omp %0' in the body of a replayed 'target data' construct is not "
+ "part of the taskgraph; it queues device work from a task the taskgraph "
+ "cannot see">,
+ InGroup<OpenMPTaskgraph>;
+def warn_omp_taskgraph_body_escaping_depend : Warning<
+ "dependence on %0 in the body of a replayed 'target data' construct does "
+ "not order the construct with tasks outside it">,
+ InGroup<OpenMPTaskgraph>;
+def note_omp_taskgraph_body : Note<
+ "body of this 'target data' construct is a task of its own">;
def note_omp_nowait_clause_here : Note<
"'nowait' clause is here">;
def err_omp_single_decl_in_declare_simd_variant : Error<
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index d2464dbc1b215c..c5beeea4af2ad7 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1323,6 +1323,19 @@ llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction(
return OutlinedFn;
}
+/// The captured region of \p D that is outlined as the body of an explicit
+/// task. Besides the tasking directives, this is the structured block of a
+/// 'target data' that a taskgraph records, which Sema gives task parameters in
+/// its only (OMPD_unknown) region.
+static OpenMPDirectiveKind
+getTaskCaptureRegion(const OMPExecutableDirective &D) {
+ if (isOpenMPTaskLoopDirective(D.getDirectiveKind()))
+ return OMPD_taskloop;
+ if (D.getDirectiveKind() == OMPD_target_data)
+ return OMPD_unknown;
+ return OMPD_task;
+}
+
llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
const VarDecl *PartIDVar, const VarDecl *TaskTVar,
@@ -1346,10 +1359,7 @@ llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction(
CodeGen.setAction(Action);
assert(!ThreadIDVar->getType()->isPointerType() &&
"thread id variable must be of type kmp_int32 for tasks");
- const OpenMPDirectiveKind Region =
- isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop
- : OMPD_task;
- const CapturedStmt *CS = D.getCapturedStmt(Region);
+ const CapturedStmt *CS = D.getCapturedStmt(getTaskCaptureRegion(D));
bool HasCancel = false;
if (const auto *TD = dyn_cast<OMPTaskDirective>(&D))
HasCancel = TD->hasCancel();
@@ -4099,10 +4109,7 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
ASTContext &C = CGF.getContext();
auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
- OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
- ? OMPD_taskloop
- : OMPD_task;
- const CapturedStmt &CS = *D.getCapturedStmt(Kind);
+ const CapturedStmt &CS = *D.getCapturedStmt(getTaskCaptureRegion(D));
CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);
LValue SrcBase;
bool IsTargetTask =
@@ -4448,7 +4455,7 @@ static llvm::Value *emitTaskCloneFunction(
LValue DstBase = PrivatesOf(DstArg);
LValue SrcBase = PrivatesOf(SrcArg);
CodeGenFunction::CGCapturedStmtInfo CapturesInfo(
- *D.getCapturedStmt(OMPD_task));
+ *D.getCapturedStmt(getTaskCaptureRegion(D)));
auto FI = PrivatesFI->getType()->castAsRecordDecl()->field_begin();
for (const PrivateDataTy &Pair : Privates) {
const FieldDecl *Field = *FI++;
@@ -4740,6 +4747,7 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
FreeAgentFlag = 0x80,
TransparentFlag = 0x100,
TaskgraphDestructorsFlag = 0x200,
+ TaskgraphBodyFlag = 0x400,
};
unsigned Flags = Data.Tied ? TiedFlag : 0;
bool NeedsCleanup = false;
@@ -4759,6 +4767,8 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
}
if (D.getSingleClause<OMPTransparentClause>())
Flags |= TransparentFlag;
+ if (Data.TaskgraphBody)
+ Flags |= TaskgraphBodyFlag;
if (Data.Priority.getInt())
Flags = Flags | PriorityFlag;
@@ -12832,12 +12842,12 @@ void CGOpenMPRuntime::emitTaskgraphTargetCall(
}
void CGOpenMPRuntime::emitTaskgraphTargetDataCall(
- CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc,
- llvm::Value *DeviceID, unsigned NumTargetItems,
- llvm::Value *BasePointersArray, llvm::Value *PointersArray,
- llvm::Value *SizesArray, llvm::Value *MapTypesArray,
- llvm::Value *MapNamesArray, llvm::Value *MappersArray,
- llvm::Value *Relocate) {
+ CodeGenFunction &CGF, const OMPExecutableDirective &D,
+ OpenMPDirectiveKind ActionKind, SourceLocation Loc, llvm::Value *DeviceID,
+ unsigned NumTargetItems, llvm::Value *BasePointersArray,
+ llvm::Value *PointersArray, llvm::Value *SizesArray,
+ llvm::Value *MapTypesArray, llvm::Value *MapNamesArray,
+ llvm::Value *MappersArray, llvm::Value *Relocate) {
llvm::Value *ThreadID = getThreadID(CGF, Loc);
llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
@@ -12863,7 +12873,7 @@ void CGOpenMPRuntime::emitTaskgraphTargetDataCall(
llvm::Value *PointerNum = CGF.Builder.getInt32(NumTargetItems);
RuntimeFunction RTLFn;
- switch (D.getDirectiveKind()) {
+ switch (ActionKind) {
case OMPD_target_enter_data:
RTLFn = OMPRTL___kmpc_taskgraph_target_enter_data;
break;
@@ -12874,7 +12884,7 @@ void CGOpenMPRuntime::emitTaskgraphTargetDataCall(
RTLFn = OMPRTL___kmpc_taskgraph_target_update;
break;
default:
- llvm_unreachable("Unexpected standalone target data directive.");
+ llvm_unreachable("Unexpected target data action.");
}
llvm::Value *Args[] = {UpLoc, ThreadID, NumDeps,
@@ -13691,9 +13701,12 @@ void CGOpenMPRuntime::emitThreadLimitClause(CodeGenFunction &CGF,
void CGOpenMPRuntime::emitTargetDataCalls(
CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
const Expr *Device, const RegionCodeGenTy &CodeGen,
- CGOpenMPRuntime::TargetDataInfo &Info) {
+ CGOpenMPRuntime::TargetDataInfo &Info, const Expr *ReplayableCond,
+ llvm::function_ref<void(CodeGenFunction &)> BodyTaskGen) {
if (!CGF.HaveInsertPoint())
return;
+ assert((!ReplayableCond || BodyTaskGen) &&
+ "a recorded 'target data' needs its body emitted as a task");
// Action used to replace the default codegen action and turn privatization
// off.
@@ -13778,15 +13791,158 @@ void CGOpenMPRuntime::emitTargetDataCalls(
// Source location for the ident struct
llvm::Value *RTLoc = emitUpdateLocation(CGF, D.getBeginLoc());
- InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator());
- InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint());
- llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CGF.Builder);
- llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
- cantFail(OMPBuilder.createTargetData(
- OmpLoc, AllocaIP, CodeGenIP, /*DeallocBlocks=*/{}, DeviceID,
- IfCondVal, Info, GenMapInfoCB, CustomMapperCB,
- /*MapperFunc=*/nullptr, BodyCB, DeviceAddrCB, RTLoc));
- CGF.Builder.restoreIP(AfterIP);
+ auto &&EmitOrdinary = [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ InsertPointTy AllocaIP(CGF.AllocaInsertPt->getIterator());
+ InsertPointTy CodeGenIP(CGF.Builder.GetInsertPoint());
+ llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CGF.Builder);
+ llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
+ cantFail(OMPBuilder.createTargetData(
+ OmpLoc, AllocaIP, CodeGenIP, /*DeallocBlocks=*/{}, DeviceID,
+ IfCondVal, Info, GenMapInfoCB, CustomMapperCB,
+ /*MapperFunc=*/nullptr, BodyCB, DeviceAddrCB, RTLoc));
+ CGF.Builder.restoreIP(AfterIP);
+ };
+
+ if (!ReplayableCond) {
+ RegionCodeGenTy RCG(EmitOrdinary);
+ RCG(CGF);
+ return;
+ }
+
+ // OpenMP 6.1 [21.5] gives 'target data' the semantics of an enter-data
+ // task, a task whose region is the body and which shares by default, and an
+ // exit-data task, in that order. All three are recorded. The body task is
+ // opaque to the taskgraph: its region is not part of the one being recorded,
+ // so what it encounters runs afresh on every replay rather than being
+ // recorded as nodes of its own.
+ auto &&EmitRecording = [&](CodeGenFunction &CGF, PrePostActionTy &) {
+ // Built once, before either action, so the stores dominate both call sites
+ // and the exit action can reuse the same argument pointers. The begin and
+ // end mapper calls of the ordinary path share these arrays too; only the
+ // map types differ, and only by dropping `present` for the end.
+ MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
+ MappableExprsHandler MEHandler(D, CGF);
+ genMapInfo(MEHandler, CGF, CombinedInfo, OMPBuilder);
+ // The body task answers its use-device list items itself, when it starts.
+ // Left in, the flag would have the begin mapper write the device address
+ // over the recorded base-pointer array, which every replay of both actions
+ // reads again.
+ for (auto &Type : CombinedInfo.Types)
+ Type &= ~OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;
+ emitOffloadingArraysAndArgs(CGF, CombinedInfo, Info, OMPBuilder,
+ /*IsNonContiguous=*/true,
+ /*ForEndCall=*/false);
+
+ // A replay may run from a different frame than the encounter that recorded
+ // the actions, so the recorded map arrays need refreshing before they are
+ // used again. One helper serves both actions: they share the arrays, and
+ // differ only in the map types, which relocation does not touch.
+ llvm::Function *RelocateFn = emitTargetRelocationFunction(
+ CGM, D.getBeginLoc(), /*CS=*/nullptr, CGF.CapturedStmtInfo,
+ CombinedInfo, /*IsDataNode=*/true);
+ llvm::Value *Relocate = RelocateFn
+ ? cast<llvm::Value>(RelocateFn)
+ : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+
+ llvm::Value *ExitMapTypesArray = Info.RTArgs.MapTypesArray;
+ if (Info.RTArgs.MapTypesArrayEnd)
+ ExitMapTypesArray = CGF.Builder.CreateConstInBoundsGEP2_32(
+ llvm::ArrayType::get(CGF.Int64Ty, Info.NumberOfPtrs),
+ Info.RTArgs.MapTypesArrayEnd, /*Idx0=*/0, /*Idx1=*/0);
+
+ auto EmitAction = [&](OpenMPDirectiveKind ActionKind,
+ llvm::Value *MapTypesArray) {
+ // As on the ordinary path, an `if` clause suppresses the mapping while
+ // leaving the body alone, so only the actions are guarded. IfCondVal was
+ // evaluated once above; the clause must not be re-evaluated here.
+ llvm::BasicBlock *ContBB = nullptr;
+ if (IfCondVal) {
+ llvm::BasicBlock *ThenBB = CGF.createBasicBlock("omp_if.then");
+ ContBB = CGF.createBasicBlock("omp_if.end");
+ CGF.Builder.CreateCondBr(IfCondVal, ThenBB, ContBB);
+ CGF.EmitBlock(ThenBB);
+ }
+ emitTaskgraphTargetDataCall(
+ CGF, D, ActionKind, D.getBeginLoc(), DeviceID, Info.NumberOfPtrs,
+ Info.RTArgs.BasePointersArray, Info.RTArgs.PointersArray,
+ Info.RTArgs.SizesArray, MapTypesArray, Info.RTArgs.MapNamesArray,
+ Info.RTArgs.MappersArray, Relocate);
+ if (ContBB) {
+ CGF.EmitBranch(ContBB);
+ CGF.EmitBlock(ContBB, /*IsFinished=*/true);
+ }
+ };
+
+ EmitAction(OMPD_target_enter_data, Info.RTArgs.MapTypesArray);
+ BodyTaskGen(CGF);
+ EmitAction(OMPD_target_exit_data, ExitMapTypesArray);
+ };
+
+ emitIfClause(CGF, ReplayableCond, EmitRecording, EmitOrdinary);
+}
+
+void CGOpenMPRuntime::emitTaskgraphUseDeviceLookup(
+ CodeGenFunction &CGF, const OMPExecutableDirective &D,
+ llvm::DenseMap<const ValueDecl *, llvm::Value *> &DeviceAddrs) {
+ // Rebuilt from the list items as they stand now: the begin mapper would see
+ // the same entries, so the answers are the ones it would have given.
+ MappableExprsHandler::MapCombinedInfoTy CombinedInfo;
+ MappableExprsHandler MEHandler(D, CGF);
+ MEHandler.generateAllInfo(CombinedInfo, OMPBuilder);
+ SmallVector<unsigned, 4> Entries;
+ for (unsigned I = 0, E = CombinedInfo.DevicePtrDecls.size(); I < E; ++I)
+ if (CombinedInfo.DevicePtrDecls[I])
+ Entries.push_back(I);
+ if (Entries.empty())
+ return;
+
+ ASTContext &C = CGM.getContext();
+ const unsigned N = Entries.size();
+ QualType PtrsTy =
+ C.getConstantArrayType(C.VoidPtrTy, llvm::APInt(32, N),
+ /*SizeExpr=*/nullptr, ArraySizeModifier::Normal,
+ /*IndexTypeQuals=*/0);
+ QualType TypesTy = C.getConstantArrayType(
+ C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true),
+ llvm::APInt(32, N), /*SizeExpr=*/nullptr, ArraySizeModifier::Normal,
+ /*IndexTypeQuals=*/0);
+ Address Bases = CGF.CreateMemTemp(PtrsTy, ".tg_use_device_bases");
+ Address Ptrs = CGF.CreateMemTemp(PtrsTy, ".tg_use_device_ptrs");
+ Address Types = CGF.CreateMemTemp(TypesTy, ".tg_use_device_types");
+ Address Out = CGF.CreateMemTemp(PtrsTy, ".tg_use_device_out");
+ for (unsigned K = 0; K < N; ++K) {
+ unsigned I = Entries[K];
+ CGF.Builder.CreateStore(CombinedInfo.BasePointers[I],
+ CGF.Builder.CreateConstArrayGEP(Bases, K));
+ CGF.Builder.CreateStore(CombinedInfo.Pointers[I],
+ CGF.Builder.CreateConstArrayGEP(Ptrs, K));
+ CGF.Builder.CreateStore(
+ CGF.Builder.getInt64(
+ static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
+ CombinedInfo.Types[I])),
+ CGF.Builder.CreateConstArrayGEP(Types, K));
+ DeviceAddrs.try_emplace(
+ CombinedInfo.DevicePtrDecls[I],
+ CGF.Builder.CreateConstArrayGEP(Out, K).emitRawPointer(CGF));
+ }
+
+ llvm::Value *DeviceID;
+ if (const auto *C = D.getSingleClause<OMPDeviceClause>())
+ DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(C->getDevice()),
+ CGF.Int64Ty, /*isSigned=*/true);
+ else
+ DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
+ llvm::Value *Args[] = {
+ DeviceID,
+ CGF.Builder.getInt32(N),
+ CGF.Builder.CreateConstArrayGEP(Bases, 0).emitRawPointer(CGF),
+ CGF.Builder.CreateConstArrayGEP(Ptrs, 0).emitRawPointer(CGF),
+ CGF.Builder.CreateConstArrayGEP(Types, 0).emitRawPointer(CGF),
+ CGF.Builder.CreateConstArrayGEP(Out, 0).emitRawPointer(CGF)};
+ CGF.EmitRuntimeCall(
+ OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___tgt_taskgraph_lookup_use_device),
+ Args);
}
void CGOpenMPRuntime::emitTargetDataStandAloneCall(
@@ -13831,7 +13987,8 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
// entry point satisfies the dependences itself; there is no enclosing
// helper task on this path.
emitTaskgraphTargetDataCall(
- CGF, D, D.getBeginLoc(), DeviceID, InputInfo.NumberOfTargetItems,
+ CGF, D, D.getDirectiveKind(), D.getBeginLoc(), DeviceID,
+ InputInfo.NumberOfTargetItems,
InputInfo.BasePointersArray.emitRawPointer(CGF),
InputInfo.PointersArray.emitRawPointer(CGF),
InputInfo.SizesArray.emitRawPointer(CGF), MapTypesArray,
@@ -15528,7 +15685,8 @@ void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF,
void CGOpenMPSIMDRuntime::emitTargetDataCalls(
CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
const Expr *Device, const RegionCodeGenTy &CodeGen,
- CGOpenMPRuntime::TargetDataInfo &Info) {
+ CGOpenMPRuntime::TargetDataInfo &Info, const Expr *ReplayableCond,
+ llvm::function_ref<void(CodeGenFunction &)> BodyTaskGen) {
llvm_unreachable("Not supported in SIMD-only mode");
}
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index aadc12ca2cad20..f15729a2b8a57a 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -127,6 +127,8 @@ struct OMPTaskDataTy final {
bool IsWorksharingReduction = false;
bool HasNowaitClause = false;
bool HasModifier = false;
+ /// The task is the structured block of a recorded 'target data'.
+ bool TaskgraphBody = false;
const Expr *ReplayableCond = nullptr;
/// The task may be recorded into a taskgraph, whose replays run its recorded
/// descriptor again, so running the body must leave the descriptor as it
@@ -1507,23 +1509,27 @@ class CGOpenMPRuntime {
llvm::Value *KernelArgsPtr,
llvm::Value *Relocate);
- /// Record a standalone 'target {enter|exit} data' / 'target update' directive
- /// nested inside a taskgraph by emitting the matching
- /// __kmpc_taskgraph_target_{enter,exit}_data / _update call. The map-info
- /// arrays are passed directly, and depend clauses of \p D are forwarded to
- /// the entry point, which satisfies them synchronously; a replayable
- /// target-data directive is never wrapped in a hidden helper task.
+ /// Record a data-mapping action nested inside a taskgraph by emitting the
+ /// matching __kmpc_taskgraph_target_{enter,exit}_data / _update call. The
+ /// map-info arrays are passed directly, and depend clauses of \p D are
+ /// forwarded to the entry point, which satisfies them synchronously; a
+ /// replayable target-data directive is never wrapped in a hidden helper task.
+ ///
+ /// \p ActionKind names the action to record. For the standalone directives it
+ /// is the directive kind itself; a 'target data' block is recorded as two
+ /// actions around its body, so it passes OMPD_target_enter_data and then
+ /// OMPD_target_exit_data.
///
/// \p Relocate refreshes the recorded map arrays for a replay from a
/// different frame, or is null where the maps cannot be re-derived; it is
/// built by the caller, which is where the map info the decision needs is.
void emitTaskgraphTargetDataCall(
- CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc,
- llvm::Value *DeviceID, unsigned NumTargetItems,
- llvm::Value *BasePointersArray, llvm::Value *PointersArray,
- llvm::Value *SizesArray, llvm::Value *MapTypesArray,
- llvm::Value *MapNamesArray, llvm::Value *MappersArray,
- llvm::Value *Relocate);
+ CodeGenFunction &CGF, const OMPExecutableDirective &D,
+ OpenMPDirectiveKind ActionKind, SourceLocation Loc, llvm::Value *DeviceID,
+ unsigned NumTargetItems, llvm::Value *BasePointersArray,
+ llvm::Value *PointersArray, llvm::Value *SizesArray,
+ llvm::Value *MapTypesArray, llvm::Value *MapNamesArray,
+ llvm::Value *MappersArray, llvm::Value *Relocate);
/// Emit the target regions enclosed in \a GD function definition or
/// the function itself in case it is a valid device function. Returns true if
@@ -1601,11 +1607,27 @@ class CGOpenMPRuntime {
/// target directive, or null if no device clause is used.
/// \param Info A record used to store information that needs to be preserved
/// until the region is closed.
- virtual void emitTargetDataCalls(CodeGenFunction &CGF,
- const OMPExecutableDirective &D,
- const Expr *IfCond, const Expr *Device,
- const RegionCodeGenTy &CodeGen,
- CGOpenMPRuntime::TargetDataInfo &Info);
+ /// \param ReplayableCond Condition under which the construct is recorded into
+ /// a taskgraph as an enter-data and an exit-data node around its body, or
+ /// null if it cannot be.
+ /// \param BodyTaskGen Emits the body as the task recorded between the two
+ /// data nodes; used only when \p ReplayableCond is non-null.
+ virtual void emitTargetDataCalls(
+ CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond,
+ const Expr *Device, const RegionCodeGenTy &CodeGen,
+ CGOpenMPRuntime::TargetDataInfo &Info,
+ const Expr *ReplayableCond = nullptr,
+ llvm::function_ref<void(CodeGenFunction &)> BodyTaskGen = nullptr);
+
+ /// Answer the use_device_ptr / use_device_addr list items of \p D from the
+ /// device data environment as it currently stands, for the body task of a
+ /// recorded 'target data', which cannot take them from a begin-mapper call
+ /// of its own. \p DeviceAddrs receives, for each list item, the address of
+ /// a slot holding its device base, as TargetDataInfo::CaptureDeviceAddrMap
+ /// does for the ordinary lowering.
+ void emitTaskgraphUseDeviceLookup(
+ CodeGenFunction &CGF, const OMPExecutableDirective &D,
+ llvm::DenseMap<const ValueDecl *, llvm::Value *> &DeviceAddrs);
/// Emit the data mapping/movement code associated with the directive
/// \a D that should be of the form 'target [{enter|exit} data | update]'.
@@ -2412,7 +2434,10 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
void emitTargetDataCalls(CodeGenFunction &CGF,
const OMPExecutableDirective &D, const Expr *IfCond,
const Expr *Device, const RegionCodeGenTy &CodeGen,
- CGOpenMPRuntime::TargetDataInfo &Info) override;
+ CGOpenMPRuntime::TargetDataInfo &Info,
+ const Expr *ReplayableCond = nullptr,
+ llvm::function_ref<void(CodeGenFunction &)>
+ BodyTaskGen = nullptr) override;
/// Emit the data mapping/movement code associated with the directive
/// \a D that should be of the form 'target [{enter|exit} data | update]'.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 504abdba73666b..e589bdb40af5e2 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -8547,9 +8547,58 @@ void CodeGenFunction::EmitOMPTargetDataDirective(
// Set the action to signal privatization of device pointers.
RCG.setAction(PrivAction);
+ // Sema gives the body task parameters only where the construct can be
+ // recorded; anywhere else it stays merged into the encountering task.
+ const CapturedStmt *CS = S.getCapturedStmt(OMPD_unknown);
+ const Expr *ReplayableCond = getOMPReplayableCond(*this, S);
+ if (CS->getCapturedDecl()->getNumParams() <= 1)
+ ReplayableCond = nullptr;
+
+ // The recorded body is a task of its own, which converts the use-device
+ // list items when it starts and then waits, in a taskgroup, for the tasks
+ // it generates: the exit-data action must not run until they are done.
+ auto &&EmitBodyTask = [&S, CS](CodeGenFunction &CGF) {
+ Address CapturedStruct = CGF.GenerateCapturedStmtArgument(*CS);
+ CanQualType SharedsTy =
+ CGF.getContext().getCanonicalTagType(CS->getCapturedRecordDecl());
+ const Expr *Replayable = IntegerLiteral::Create(
+ CGF.getContext(), llvm::APInt(32, 1),
+ CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
+ SourceLocation());
+ OMPTaskDataTy Data;
+ Data.Tied = true;
+ Data.TaskgraphBody = true;
+ Data.ReplayableCond = Replayable;
+ auto &&BodyGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &) {
+ llvm::DenseMap<const ValueDecl *, llvm::Value *> DeviceAddrs;
+ CGF.CGM.getOpenMPRuntime().emitTaskgraphUseDeviceLookup(CGF, S,
+ DeviceAddrs);
+ OMPPrivateScope PrivateScope(CGF);
+ for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
+ CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope, DeviceAddrs);
+ for (const auto *C : S.getClausesOfKind<OMPUseDeviceAddrClause>())
+ CGF.EmitOMPUseDeviceAddrClause(*C, PrivateScope, DeviceAddrs);
+ (void)PrivateScope.Privatize();
+ auto &&Body = [CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
+ Action.Enter(CGF);
+ CGF.EmitStmt(CS->getCapturedStmt());
+ };
+ CGF.CGM.getOpenMPRuntime().emitTaskgroupRegion(CGF, Body,
+ S.getBeginLoc());
+ };
+ auto &&TaskGen = [&S, SharedsTy, CapturedStruct, Replayable](
+ CodeGenFunction &CGF, llvm::Function *OutlinedFn,
+ const OMPTaskDataTy &Data) {
+ CGF.CGM.getOpenMPRuntime().emitTaskCall(
+ CGF, S.getBeginLoc(), S, OutlinedFn, SharedsTy, CapturedStruct,
+ /*IfCond=*/nullptr, Replayable, Data);
+ };
+ CGF.EmitOMPTaskBasedDirective(S, OMPD_unknown, BodyGen, TaskGen, Data);
+ };
+
// Emit region code.
- CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
- Info);
+ CGM.getOpenMPRuntime().emitTargetDataCalls(
+ *this, S, IfCond, Device, RCG, Info, ReplayableCond, EmitBodyTask);
}
void CodeGenFunction::EmitOMPTargetEnterDataDirective(
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 52f510542cf69e..a661f6db5f4c00 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -208,6 +208,10 @@ class DSAStackTy {
const Decl *PossiblyLoopCounter = nullptr;
bool NowaitRegion = false;
bool UntiedRegion = false;
+ bool ReplayableRegion = false;
+ /// For a 'target data' whose structured block a taskgraph records, the
+ /// captured declaration of that block.
+ const CapturedDecl *TaskgraphBodyDecl = nullptr;
bool CancelRegion = false;
bool LoopStart = false;
bool BodyComplete = false;
@@ -965,6 +969,51 @@ class DSAStackTy {
return Parent->NowaitRegion;
return false;
}
+ /// Marks current region as replayable (it has a 'replayable' clause).
+ void setReplayableRegion() { getTopOfStack().ReplayableRegion = true; }
+ /// Whether the current directive is a 'target data' whose structured block
+ /// is the task a taskgraph records and replays: the directive is encountered
+ /// in a taskgraph region, or is marked replayable.
+ bool isTaskgraphTargetDataBody() const {
+ if (isStackEmpty())
+ return false;
+ return isTaskgraphTargetDataBody(getStackSize() - 1);
+ }
+ /// Records the captured declaration of the current directive's structured
+ /// block, when isTaskgraphTargetDataBody().
+ void setTaskgraphBodyDecl(const CapturedDecl *CD) {
+ getTopOfStack().TaskgraphBodyDecl = CD;
+ }
+ /// The innermost 'target data' enclosing the current directive whose
+ /// structured block a taskgraph records, looking out through the constructs
+ /// nested in that block but not past a taskgraph or target region, each of
+ /// which begins a region of its own. \p Loc is set to its location.
+ const CapturedDecl *getEnclosingTaskgraphBody(SourceLocation &Loc) const {
+ if (getStackSize() < 2)
+ return nullptr;
+ for (unsigned Level = getStackSize() - 1; Level-- > 0;) {
+ const SharingMapTy &Elem = getStackElemAtLevel(Level);
+ if (Elem.Directive == OMPD_taskgraph ||
+ isOpenMPTargetExecutionDirective(Elem.Directive))
+ return nullptr;
+ if (Elem.TaskgraphBodyDecl) {
+ Loc = Elem.ConstructLoc;
+ return Elem.TaskgraphBodyDecl;
+ }
+ }
+ return nullptr;
+ }
+ /// As above, for the directive at \p Level.
+ bool isTaskgraphTargetDataBody(unsigned Level) const {
+ if (Level >= getStackSize())
+ return false;
+ const SharingMapTy &Elem = getStackElemAtLevel(Level);
+ if (Elem.Directive != OMPD_target_data)
+ return false;
+ return Elem.ReplayableRegion ||
+ (Level > 0 &&
+ getStackElemAtLevel(Level - 1).Directive == OMPD_taskgraph);
+ }
/// Marks current region as untied (it has a 'untied' clause).
void setUntiedRegion(bool IsUntied = true) {
getTopOfStack().UntiedRegion = IsUntied;
@@ -2225,6 +2274,12 @@ bool SemaOpenMP::isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
if (const auto *BD = dyn_cast<BindingDecl>(D))
Ty = BD->getDecomposedDecl()->getType();
+ // The recorded body of a 'target data' is a task that shares by default,
+ // and a replay refreshes its shareds as addresses. A use-device list item,
+ // though firstprivate to the construct, is converted by the body itself.
+ if (DSAStack->isTaskgraphTargetDataBody(Level))
+ return true;
+
bool IsVariableUsedInMapClause = false;
if (DSAStack->hasExplicitDirective(isOpenMPTargetExecutionDirective, Level)) {
// This table summarizes how a given variable should be passed to the device
@@ -4942,7 +4997,8 @@ getTaskloopRegionParams(Sema &SemaRef) {
}
static void processCapturedRegions(Sema &SemaRef, OpenMPDirectiveKind DKind,
- Scope *CurScope, SourceLocation Loc) {
+ Scope *CurScope, SourceLocation Loc,
+ bool TaskgraphTargetDataBody) {
SmallVector<OpenMPDirectiveKind> Regions;
getOpenMPCaptureRegions(Regions, DKind);
@@ -4989,6 +5045,17 @@ static void processCapturedRegions(Sema &SemaRef, OpenMPDirectiveKind DKind,
getTargetRegionParams(SemaRef), Level);
break;
case OMPD_unknown:
+ // OpenMP 6.1 [21.5] makes the structured block of a 'target data' a task
+ // of its own, bracketed by the enter-data and exit-data tasks. A
+ // taskgraph records and replays that task, so it is given the parameters
+ // of one; everywhere else it is merged into the encountering task and
+ // the block is emitted inline.
+ if (TaskgraphTargetDataBody) {
+ SemaRef.ActOnCapturedRegionStart(Loc, CurScope, CR_OpenMP,
+ getTaskRegionParams(SemaRef));
+ MarkAsInlined(SemaRef.getCurCapturedRegion());
+ break;
+ }
SemaRef.ActOnCapturedRegionStart(Loc, CurScope, CR_OpenMP,
getUnknownRegionParams(SemaRef));
break;
@@ -5024,7 +5091,11 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
break;
default:
processCapturedRegions(SemaRef, DKind, CurScope,
- DSAStack->getConstructLoc());
+ DSAStack->getConstructLoc(),
+ DSAStack->isTaskgraphTargetDataBody());
+ if (DSAStack->isTaskgraphTargetDataBody())
+ DSAStack->setTaskgraphBodyDecl(
+ SemaRef.getCurCapturedRegion()->TheCapturedDecl);
break;
}
@@ -5278,6 +5349,25 @@ StmtResult SemaOpenMP::ActOnOpenMPRegionEnd(StmtResult S,
else if (Clause->getClauseKind() == OMPC_linear)
LCs.push_back(cast<OMPLinearClause>(Clause));
}
+ // The structured block of a replayed 'target data' is a task that converts
+ // the use-device list items itself when it starts, from the map arrays it
+ // rebuilds, so everything those arrays are built from is shared with it.
+ if (DSAStack->isTaskgraphTargetDataBody()) {
+ for (OMPClause *Clause : Clauses) {
+ switch (Clause->getClauseKind()) {
+ case OMPC_map:
+ case OMPC_use_device_ptr:
+ case OMPC_use_device_addr:
+ case OMPC_device:
+ for (Stmt *Child : Clause->children())
+ if (auto *E = dyn_cast_or_null<Expr>(Child))
+ SemaRef.MarkDeclarationsReferencedInExpr(E);
+ break;
+ default:
+ break;
+ }
+ }
+ }
// Capture allocator expressions if used.
for (Expr *E : DSAStack->getInnerAllocators())
SemaRef.MarkDeclarationsReferencedInExpr(E);
@@ -6780,6 +6870,83 @@ static bool teamsLoopCanBeParallelFor(Stmt *AStmt, Sema &SemaRef) {
return Checker.teamsLoopCanBeParallelFor();
}
+/// The variable a depend list item is a part of, or null if it is not a part
+/// of a variable (an lvalue reached through a pointer is a part of the
+/// pointer's variable, as far as this lexical check can tell).
+static const VarDecl *getDependBaseDecl(const Expr *E) {
+ while (true) {
+ E = E->IgnoreParenImpCasts();
+ if (const auto *OASE = dyn_cast<ArraySectionExpr>(E))
+ E = OASE->getBase();
+ else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(E))
+ E = ASE->getBase();
+ else if (const auto *ME = dyn_cast<MemberExpr>(E))
+ E = ME->getBase();
+ else if (const auto *Shape = dyn_cast<OMPArrayShapingExpr>(E))
+ E = Shape->getBase();
+ else if (const auto *UO = dyn_cast<UnaryOperator>(E);
+ UO && UO->getOpcode() == UO_Deref)
+ E = UO->getSubExpr();
+ else
+ break;
+ }
+ if (const auto *DRE = dyn_cast<DeclRefExpr>(E))
+ return dyn_cast<VarDecl>(DRE->getDecl());
+ return nullptr;
+}
+
+/// OpenMP 6.1 [21.5]: the structured block of a 'target data' that a taskgraph
+/// records is a task of its own, and the region it executes is not part of
+/// the taskgraph region. Two things written in it therefore do not do what
+/// they appear to, and are diagnosed here, lexically:
+///
+/// * a device construct is not recorded, so it queues device work from a
+/// host task the taskgraph cannot see;
+/// * a dependence on storage declared outside the block orders nothing
+/// outside it, since only the construct's own tasks are in the graph.
+static void checkTaskgraphTargetDataBody(Sema &SemaRef, DSAStackTy *Stack,
+ OpenMPDirectiveKind Kind,
+ ArrayRef<OMPClause *> Clauses,
+ SourceLocation StartLoc) {
+ SourceLocation BodyLoc;
+ const CapturedDecl *Body = Stack->getEnclosingTaskgraphBody(BodyLoc);
+ if (!Body)
+ return;
+ if (isOpenMPTargetExecutionDirective(Kind) || Kind == OMPD_target_data ||
+ Kind == OMPD_target_enter_data || Kind == OMPD_target_exit_data ||
+ Kind == OMPD_target_update) {
+ SemaRef.Diag(StartLoc, diag::warn_omp_taskgraph_body_device_construct)
+ << getOpenMPDirectiveName(Kind,
+ SemaRef.getLangOpts().getOpenMPVersion());
+ SemaRef.Diag(BodyLoc, diag::note_omp_taskgraph_body);
+ }
+ for (const OMPClause *C : Clauses) {
+ const auto *DC = dyn_cast_or_null<OMPDependClause>(C);
+ if (!DC || DC->getModifier())
+ continue;
+ OpenMPDependClauseKind DepKind = DC->getDependencyKind();
+ if (DepKind == OMPC_DEPEND_depobj || DepKind == OMPC_DEPEND_source ||
+ DepKind == OMPC_DEPEND_sink || DepKind == OMPC_DEPEND_outallmemory ||
+ DepKind == OMPC_DEPEND_inoutallmemory)
+ continue;
+ for (const Expr *E : DC->varlist()) {
+ if (!E || E->isTypeDependent())
+ continue;
+ QualType DependT = Stack->getOMPDependT();
+ if (!DependT.isNull() &&
+ SemaRef.getASTContext().hasSameUnqualifiedType(E->getType(), DependT))
+ continue;
+ const VarDecl *VD = getDependBaseDecl(E);
+ if (!VD || Body->Encloses(VD->getDeclContext()))
+ continue;
+ SemaRef.Diag(E->getExprLoc(),
+ diag::warn_omp_taskgraph_body_escaping_depend)
+ << VD;
+ SemaRef.Diag(BodyLoc, diag::note_omp_taskgraph_body);
+ }
+ }
+}
+
StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
OpenMPDirectiveKind Kind, const DeclarationNameInfo &DirName,
OpenMPDirectiveKind CancelRegion, ArrayRef<OMPClause *> Clauses,
@@ -6841,6 +7008,9 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
return StmtError();
}
+ if (!SemaRef.inTemplateInstantiation())
+ checkTaskgraphTargetDataBody(SemaRef, DSAStack, Kind, Clauses, StartLoc);
+
// Report affected OpenMP target offloading behavior when in HIP lang-mode.
if (getLangOpts().HIP && (isOpenMPTargetExecutionDirective(Kind) ||
isOpenMPTargetDataManagementDirective(Kind)))
@@ -20608,6 +20778,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPReplayableClause(SourceLocation StartLoc,
ValExpr = Val.get();
}
}
+ DSAStack->setReplayableRegion();
return new (getASTContext())
OMPReplayableClause(ValExpr, StartLoc, LParenLoc, EndLoc);
}
diff --git a/clang/test/OpenMP/taskgraph_target_data_block_codegen.cpp b/clang/test/OpenMP/taskgraph_target_data_block_codegen.cpp
new file mode 100644
index 00000000000000..9a9e99945d39c8
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_target_data_block_codegen.cpp
@@ -0,0 +1,155 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-linux-gnu -fopenmp-targets=x86_64-unknown-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// Verifies that the 'omp target data' block construct is recorded into a
+// taskgraph as three nodes: an enter-data action, a task whose region is the
+// body, and an exit-data action. The body task carries the taskgraph-body
+// flag (0x400, so 1025 with tiedness), runs its region inside a taskgroup, and
+// converts its use-device list items itself when it starts. Outside a
+// taskgraph, and when replayable(false) opts out, the construct still lowers
+// to the ordinary __tgt_target_data_{begin,end}_mapper pair with the body
+// inline.
+//
+// The standalone 'target {enter|exit} data' / 'target update' directives are
+// covered by taskgraph_target_data_codegen.cpp.
+
+void outside() {
+ int x = 0;
+#pragma omp target data map(tofrom : x)
+ { x += 1; }
+}
+
+void inside() {
+ int x = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : x)
+ { x += 1; }
+ }
+}
+
+// The use-device list item is answered by the body task when it starts, so
+// the recorded actions do not ask for it: RETURN_PARAM (0x40) is dropped from
+// their map types, leaving 0x3.
+void usedev(int *p, int n) {
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : p[0 : n]) use_device_ptr(p)
+ { p[0] = 0; }
+ }
+}
+
+// A 'replayable' clause records the construct outside a taskgraph too.
+void replayable_clause() {
+ int x = 0;
+#pragma omp target data map(tofrom : x) replayable
+ { x += 1; }
+}
+
+// A 'present' modifier is dropped from the exit action's map types, exactly as
+// it is from the ordinary end-mapper call, so the two actions pass different
+// map-type arrays.
+void present() {
+ int x = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target data map(present, tofrom : x)
+ { x += 1; }
+ }
+}
+
+// An 'if' clause suppresses the mapping while leaving the body alone, so it
+// guards each action and not the body.
+void ifclause(int c) {
+ int x = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : x) if (c)
+ { x += 1; }
+ }
+}
+
+void optout() {
+ int x = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : x) replayable(0)
+ { x += 1; }
+ }
+}
+
+// The map types of the 'present' case: 0x1003 (present|to|from) for the enter
+// action, and 0x3 for the exit action, which drops 'present'.
+// CHECK: @[[MT_USEDEV:[.0-9A-Za-z_]+]] = private unnamed_addr constant [2 x i64] [i64 3, i64 16384]
+// CHECK: @[[MT_PRESENT:[.0-9A-Za-z_]+]] = private unnamed_addr constant [1 x i64] [i64 4099]
+// CHECK-NEXT: @[[MT_END:[.0-9A-Za-z_]+]] = private unnamed_addr constant [1 x i64] [i64 3]
+
+// Outside a taskgraph: the ordinary mapper pair, no taskgraph recording.
+// CHECK-LABEL: define {{.*}}@_Z7outsidev
+// CHECK: call void @__tgt_target_data_begin_mapper(
+// CHECK: call void @__tgt_target_data_end_mapper(
+// CHECK-NOT: call void @__kmpc_taskgraph_target_
+
+// Inside a taskgraph: an enter-data node, the body task, then an exit-data
+// node. With no depend clause, ndeps (arg 3) is 0 and dep_list (arg 4) is
+// null; the seven map-info pointers are forwarded, followed by the relocation
+// callback. Both actions share the base-pointer and pointer arrays, which are
+// filled in once ahead of the enter action so that they dominate both calls --
+// and so share one callback too, there being one pair of arrays to refresh.
+// The body task has a relocation helper of its own, for its shareds.
+// CHECK-LABEL: define {{.*}}@_Z6insidev
+// CHECK-LABEL: define internal void @taskgraph.omp_outlined.(
+// CHECK: call void @__kmpc_taskgraph_target_enter_data(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 1, ptr [[BP:%[^,]+]], ptr [[P:%[^,]+]], ptr {{[^,]+}}, ptr {{[^,]+}}, ptr null, ptr null, ptr [[RELOC:@[.0-9A-Za-z_]+]])
+// CHECK: call ptr @__kmpc_omp_task_alloc(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 1025, i64 {{[0-9]+}}, i64 {{[0-9]+}}, ptr [[BODY:@[.0-9A-Za-z_]+]])
+// CHECK: call i32 @__kmpc_taskgraph_task(ptr {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, i32 1025, i64 {{[0-9]+}}, i64 {{[0-9]+}}, i32 0, ptr null, ptr @.omp.taskgraph.relocate.{{[.0-9]*}}, ptr null)
+// CHECK: call void @__kmpc_taskgraph_target_exit_data(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 0, ptr null, i32 0, i64 {{[^,]+}}, i32 1, ptr [[BP]], ptr [[P]], ptr {{[^,]+}}, ptr {{[^,]+}}, ptr null, ptr null, ptr [[RELOC]])
+// CHECK-NOT: call void @__tgt_target_data_begin_mapper(
+// CHECK: define internal {{.*}}i32 [[BODY]](
+// CHECK-NOT: __tgt_taskgraph_lookup_use_device
+// CHECK: call void @__kmpc_taskgroup(
+// CHECK: add nsw i32
+// CHECK: call void @__kmpc_end_taskgroup(
+
+// CHECK-LABEL: define {{.*}}@_Z6usedevPii
+// CHECK-LABEL: define internal void @taskgraph.omp_outlined..{{[0-9]+}}(
+// CHECK: call void @__kmpc_taskgraph_target_enter_data({{.*}}, ptr @[[MT_USEDEV]], ptr null, ptr null, ptr @{{[.0-9A-Za-z_]+}})
+// CHECK: call ptr @__kmpc_omp_task_alloc(ptr {{[^,]+}}, i32 {{[^,]+}}, i32 1025, i64 {{[0-9]+}}, i64 {{[0-9]+}}, ptr [[UDBODY:@[.0-9A-Za-z_]+]])
+// CHECK: call void @__kmpc_taskgraph_target_exit_data({{.*}}, ptr @[[MT_USEDEV]], ptr null, ptr null, ptr @{{[.0-9A-Za-z_]+}})
+// CHECK: define internal {{.*}}i32 [[UDBODY]](
+// CHECK: store i64 67, ptr
+// CHECK: call void @__tgt_taskgraph_lookup_use_device(i64 -1, i32 1, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr [[OUT:%[^,)]+]])
+// CHECK: call void @__kmpc_taskgroup(
+// CHECK: load ptr, ptr [[OUT]]
+// CHECK: call void @__kmpc_end_taskgroup(
+
+// CHECK-LABEL: define {{.*}}@_Z17replayable_clausev
+// CHECK: call void @__kmpc_taskgraph_target_enter_data(
+// CHECK: call i32 @__kmpc_taskgraph_task(ptr {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, i32 1025,
+// CHECK: call void @__kmpc_taskgraph_target_exit_data(
+
+// CHECK-LABEL: define {{.*}}@_Z7presentv
+// CHECK: call void @__kmpc_taskgraph_target_enter_data({{.*}}, ptr @[[MT_PRESENT]], ptr null, ptr null, ptr @{{[.0-9A-Za-z_]+}})
+// CHECK: call void @__kmpc_taskgraph_target_exit_data({{.*}}, ptr @[[MT_END]], ptr null, ptr null, ptr @{{[.0-9A-Za-z_]+}})
+
+// Each action sits in its own guarded block, and both branch on the same
+// already-evaluated condition rather than re-evaluating the clause.
+// CHECK-LABEL: define {{.*}}@_Z8ifclausei
+// CHECK: [[COND:%[a-z0-9.]+]] = icmp ne i32 {{%[0-9]+}}, 0
+// CHECK: br i1 [[COND]], label %[[THEN1:[^,]+]], label %[[END1:[a-z0-9._]+]]
+// CHECK: [[THEN1]]:
+// CHECK: call void @__kmpc_taskgraph_target_enter_data(
+// CHECK: [[END1]]:
+// CHECK: br i1 [[COND]], label %[[THEN2:[^,]+]], label %{{[a-z0-9._]+}}
+// CHECK: [[THEN2]]:
+// CHECK: call void @__kmpc_taskgraph_target_exit_data(
+
+// replayable(false): the ordinary mapper pair, even inside the taskgraph.
+// CHECK-LABEL: define {{.*}}@_Z6optoutv
+// CHECK: call void @__tgt_target_data_begin_mapper(
+// CHECK: call void @__tgt_target_data_end_mapper(
+// CHECK-NOT: call void @__kmpc_taskgraph_target_
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_target_data_body_messages.cpp b/clang/test/OpenMP/taskgraph_target_data_body_messages.cpp
new file mode 100644
index 00000000000000..829073627f1a8b
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_target_data_body_messages.cpp
@@ -0,0 +1,157 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -fopenmp-targets=amdgcn-amd-amdhsa -ferror-limit 100 -o - %s
+// RUN: %clang_cc1 -verify=quiet -fopenmp -fopenmp-version=60 -fopenmp-targets=amdgcn-amd-amdhsa -Wno-openmp-taskgraph -ferror-limit 100 -o - %s
+
+// quiet-no-diagnostics
+
+// The structured block of a 'target data' recorded by a taskgraph is a task
+// of its own, whose region is not part of the taskgraph region. Device
+// constructs written in it are not recorded, and dependences on storage from
+// outside it order nothing outside it; both are diagnosed, lexically.
+
+typedef void *omp_depend_t;
+
+int Global;
+
+void device_constructs(int *A, int N) {
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : A[0 : N]) // expected-note 6 {{body of this 'target data' construct is a task of its own}}
+ {
+#pragma omp target // expected-warning {{'#pragma omp target' in the body of a replayed 'target data' construct is not part of the taskgraph}}
+ A[0] = 1;
+#pragma omp target enter data map(to : A[0 : N]) // expected-warning {{'#pragma omp target_enter_data' in the body}}
+#pragma omp target exit data map(release : A[0 : N]) // expected-warning {{'#pragma omp target_exit_data' in the body}}
+#pragma omp target update to(A[0 : N]) // expected-warning {{'#pragma omp target_update' in the body}}
+#pragma omp target data map(to : A[0 : N]) // expected-warning {{'#pragma omp target_data' in the body}}
+ {
+ A[1] = 2;
+ }
+#pragma omp task
+ {
+#pragma omp target teams // expected-warning {{'#pragma omp target teams' in the body}}
+ A[2] = 3;
+ }
+ }
+ }
+}
+
+// A 'replayable' clause makes the body a task outside a taskgraph too.
+void replayable_clause(int *A, int N) {
+#pragma omp target data map(tofrom : A[0 : N]) replayable // expected-note {{body of this 'target data' construct is a task of its own}}
+ {
+#pragma omp target // expected-warning {{'#pragma omp target' in the body}}
+ A[0] = 1;
+ }
+}
+
+// A target region inside the body starts a region of its own: what is nested
+// in it is not the body's business.
+void stops_at_target(int *A, int N) {
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : A[0 : N]) // expected-note {{body of this 'target data' construct is a task of its own}}
+ {
+#pragma omp target map(tofrom : A[0 : N]) // expected-warning {{'#pragma omp target' in the body}}
+ {
+#pragma omp parallel for
+ for (int I = 0; I < N; ++I)
+ A[I] = I;
+ }
+ }
+ }
+}
+
+void depends(int *A, int N, omp_depend_t Obj) {
+ int Outer = 0;
+ int Arr[4];
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : A[0 : N]) // expected-note 5 {{body of this 'target data' construct is a task of its own}}
+ {
+ int Inner = 0;
+ int InnerArr[4];
+#pragma omp task depend(out : Outer) // expected-warning {{dependence on 'Outer' in the body of a replayed 'target data' construct does not order the construct with tasks outside it}}
+ Outer = 1;
+#pragma omp task depend(in : Global) // expected-warning {{dependence on 'Global'}}
+ (void)Global;
+#pragma omp task depend(inout : A[0 : N]) // expected-warning {{dependence on 'A'}}
+ A[0] = 1;
+#pragma omp task depend(in : Arr[1]) // expected-warning {{dependence on 'Arr'}}
+ (void)Arr[1];
+#pragma omp taskwait depend(in : Outer) // expected-warning {{dependence on 'Outer'}}
+ // Storage declared in the body is the body's own.
+#pragma omp task depend(out : Inner)
+ Inner = 1;
+#pragma omp task depend(inout : InnerArr[0 : 2])
+ InnerArr[0] = 1;
+#pragma omp task depend(in : Inner)
+ {
+ int Nested = 0;
+#pragma omp task depend(out : Nested)
+ Nested = Inner;
+ }
+ // Dependence objects and iterators are left alone.
+#pragma omp task depend(depobj : Obj)
+ A[1] = 1;
+#pragma omp task depend(iterator(I = 0 : 4), in : Arr[I])
+ A[2] = 1;
+ }
+ }
+}
+
+// Only a body the taskgraph records is a task: outside a taskgraph, and
+// without 'replayable', the block is merged into the encountering task.
+void not_recorded(int *A, int N) {
+ int Outer = 0;
+#pragma omp target data map(tofrom : A[0 : N])
+ {
+#pragma omp target
+ A[0] = 1;
+#pragma omp task depend(out : Outer)
+ Outer = 1;
+ }
+#pragma omp taskgraph
+ {
+#pragma omp task
+ {
+#pragma omp target data map(tofrom : A[0 : N])
+ {
+#pragma omp target
+ A[0] = 1;
+ }
+ }
+ }
+}
+
+// A taskgraph nested in the body starts a region of its own.
+void stops_at_taskgraph(int *A, int N) {
+ int Outer = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : A[0 : N])
+ {
+#pragma omp taskgraph
+ {
+#pragma omp task depend(out : Outer)
+ Outer = 1;
+ }
+ }
+ }
+}
+
+template <typename T> void in_template(T *A, int N) {
+ int Outer = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : A[0 : N]) // expected-note 2 {{body of this 'target data' construct is a task of its own}}
+ {
+#pragma omp target // expected-warning {{'#pragma omp target' in the body}}
+ A[0] = 1;
+#pragma omp task depend(out : Outer) // expected-warning {{dependence on 'Outer'}}
+ Outer = 1;
+ }
+ }
+}
+
+template void in_template<int>(int *, int);
+template void in_template<float>(float *, int);
diff --git a/flang/test/Parser/OpenMP/replayable-clause.f90 b/flang/test/Parser/OpenMP/replayable-clause.f90
index 24ccc017808985..25d622a520a6bc 100644
--- a/flang/test/Parser/OpenMP/replayable-clause.f90
+++ b/flang/test/Parser/OpenMP/replayable-clause.f90
@@ -58,3 +58,29 @@ subroutine f02
!PARSE-TREE: | | LiteralConstant -> LogicalLiteralConstant
!PARSE-TREE: | | | bool = 'false'
!PARSE-TREE: | Flags = {}
+
+
+! The block form of target data takes the clause too.
+subroutine f03(x)
+ implicit none
+ integer :: x
+ !$omp target data map(tofrom: x) replayable(.true.)
+ x = x + 1
+ !$omp end target data
+end
+
+!UNPARSE: SUBROUTINE f03 (x)
+!UNPARSE: IMPLICIT NONE
+!UNPARSE: INTEGER x
+!UNPARSE: !$OMP TARGET_DATA MAP(TOFROM: x) REPLAYABLE(.true._4)
+!UNPARSE: !$OMP END TARGET_DATA
+!UNPARSE: END SUBROUTINE
+
+!PARSE-TREE: ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OmpBlockConstruct
+!PARSE-TREE: | OmpBeginDirective
+!PARSE-TREE: | | OmpDirectiveName -> llvm::omp::Directive = target data
+!PARSE-TREE: | | OmpClauseList -> OmpClause -> Map -> OmpMapClause
+!PARSE-TREE: | | OmpClause -> Replayable -> OmpReplayableClause -> Scalar -> Logical -> Constant -> Expr = '.true._4'
+!PARSE-TREE: | | | LiteralConstant -> LogicalLiteralConstant
+!PARSE-TREE: | | | | bool = 'true'
+!PARSE-TREE: | | Flags = {}
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 393efc9344d269..d2d8cf636bcf75 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -1303,6 +1303,7 @@ def OMP_TargetData : Directive<[Spelling<"target data", 1, 52>,
let allowedOnceClauses = [
VersionedClause<OMPC_Default, 60>,
VersionedClause<OMPC_Device>,
+ VersionedClause<OMPC_Replayable, 60>,
VersionedClause<OMPC_Transparent, 60>,
];
let requiredClauses = [
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 96bc290450c4c5..0e153994f5e2db 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -489,6 +489,8 @@ __OMP_RTL(__tgt_target_data_begin_nowait_mapper, false, Void, IdentPtr, Int64,
__OMP_RTL(__tgt_target_data_begin_mapper_issue, false, Void, IdentPtr, Int64, Int32,
VoidPtrPtr, VoidPtrPtr, Int64Ptr, Int64Ptr, VoidPtrPtr, VoidPtrPtr, AsyncInfoPtr)
__OMP_RTL(__tgt_target_data_begin_mapper_wait, false, Void, Int64, AsyncInfoPtr)
+__OMP_RTL(__tgt_taskgraph_lookup_use_device, false, Void, Int64, Int32,
+ VoidPtrPtr, VoidPtrPtr, Int64Ptr, VoidPtrPtr)
__OMP_RTL(__tgt_target_data_end_mapper, false, Void, IdentPtr, Int64, Int32, VoidPtrPtr,
VoidPtrPtr, Int64Ptr, Int64Ptr, VoidPtrPtr, VoidPtrPtr)
__OMP_RTL(__tgt_target_data_end_nowait_mapper, false, Void, IdentPtr, Int64,
diff --git a/offload/test/offloading/taskgraph_map_overlap.c b/offload/test/offloading/taskgraph_map_overlap.c
new file mode 100644
index 00000000000000..c3b498aa44b044
--- /dev/null
+++ b/offload/test/offloading/taskgraph_map_overlap.c
@@ -0,0 +1,147 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Two mappings of one array within a single taskgraph. Their host ranges can
+// coincide, one can contain the other, or they can be disjoint -- and a graph
+// has to reach the same answer as the same program without one, in every shape.
+//
+// What makes this worth pinning down is that a device buffer is identified by
+// its host range, and the two mappings are two *different* ranges of the same
+// array. Whether they are one buffer or two is decided by the ranges alone, so
+// nothing here needs to recognise that the ranges came from the same array: a
+// contained range must resolve to its container at an offset (exactly what
+// libomptarget's own mapping table does, reusing the enclosing entry at
+// TgtPtrBegin + (HstPtrBegin - entry->HstPtrBegin)), while disjoint ranges must
+// stay independent.
+//
+// Containment is the shape that discriminates, and the enclosing `target data`
+// is what makes it observable. The enclosing region's copy-out is the last
+// thing to touch the host range, so if the inner kernel wrote a *separate*
+// device buffer, that copy-out writes back the values the enclosing region
+// copied in before the kernel ran and the kernel's work is silently undone.
+// Laundering the result through host memory cannot rescue it: the enclosing
+// mapping is live across the kernel, so there is no copy-out/copy-in pair
+// between the two to carry the value.
+//
+// The disjoint shapes are here to keep a fix honest in the other direction: a
+// coherence fix that merged ranges by proximity rather than by overlap would
+// hand the second kernel a buffer that runs on into the first one's section,
+// so adjacent-but-disjoint must stay two buffers. Adjacent and gapped are
+// separated because only the adjacent pair is at risk of being coalesced.
+//
+// Replayed several times, so a shape that is right on the recording execution
+// and wrong from the first replay onwards is still caught.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 1024
+#define Q (N / 4)
+#define ITERS 3
+
+static int check(const char *Shape, int *Arr, int Lo, int Hi) {
+ int Errors = 0;
+ for (int I = 0; I < N; ++I) {
+ int Expected = I + (I >= Lo && I < Hi ? ITERS : 0);
+ if (Arr[I] != Expected)
+ ++Errors;
+ }
+ printf("%s errors: %d\n", Shape, Errors);
+ return Errors;
+}
+
+int main() {
+ int *A = (int *)malloc(N * sizeof(int));
+ int *B = (int *)malloc(N * sizeof(int));
+ int *C = (int *)malloc(N * sizeof(int));
+ int *D = (int *)malloc(N * sizeof(int));
+ int *E = (int *)malloc(N * sizeof(int));
+ for (int I = 0; I < N; ++I)
+ A[I] = B[I] = C[I] = D[I] = E[I] = I;
+
+ for (int It = 0; It < ITERS; ++It) {
+#pragma omp taskgraph
+ {
+ // Coinciding: the inner range is exactly the enclosing one.
+#pragma omp target data map(tofrom : A[0 : N])
+ {
+#pragma omp target map(tofrom : A[0 : N])
+ for (int I = 0; I < N; ++I)
+ A[I] += 1;
+ }
+
+ // Containing: the inner range is a strict sub-range of the enclosing
+ // one, touching neither endpoint, so it cannot be mistaken for it.
+#pragma omp target data map(tofrom : B[0 : N])
+ {
+#pragma omp target map(tofrom : B[Q : 2 * Q])
+ for (int I = 0; I < 2 * Q; ++I)
+ B[Q + I] += 1;
+ }
+
+ // Containing, sharing the enclosing range's start, which is the one
+ // sub-range an exact-pointer buffer key does resolve to the container.
+#pragma omp target data map(tofrom : C[0 : N])
+ {
+#pragma omp target map(tofrom : C[0 : 2 * Q])
+ for (int I = 0; I < 2 * Q; ++I)
+ C[I] += 1;
+ }
+
+ // Adjacent-disjoint: two ranges meeting end to start.
+#pragma omp target map(tofrom : D[0 : N / 2])
+ for (int I = 0; I < N / 2; ++I)
+ D[I] += 1;
+#pragma omp target map(tofrom : D[N / 2 : N / 2])
+ for (int I = 0; I < N / 2; ++I)
+ D[N / 2 + I] += 1;
+
+ // Gapped-disjoint: a hole between the two ranges, which must stay
+ // untouched.
+#pragma omp target map(tofrom : E[0 : Q])
+ for (int I = 0; I < Q; ++I)
+ E[I] += 1;
+#pragma omp target map(tofrom : E[2 * Q : Q])
+ for (int I = 0; I < Q; ++I)
+ E[2 * Q + I] += 1;
+ }
+ }
+
+ int Errors = 0;
+ // CHECK: coinciding errors: 0
+ Errors += check("coinciding", A, 0, N);
+ // CHECK: containing errors: 0
+ Errors += check("containing", B, Q, 3 * Q);
+ // CHECK: containing-at-start errors: 0
+ Errors += check("containing-at-start", C, 0, 2 * Q);
+ // CHECK: adjacent-disjoint errors: 0
+ Errors += check("adjacent-disjoint", D, 0, N);
+
+ int GapErrors = 0;
+ for (int I = 0; I < N; ++I) {
+ int Expected = I + (I < Q || (I >= 2 * Q && I < 3 * Q) ? ITERS : 0);
+ if (E[I] != Expected)
+ ++GapErrors;
+ }
+ // CHECK: gapped-disjoint errors: 0
+ printf("gapped-disjoint errors: %d\n", GapErrors);
+ Errors += GapErrors;
+
+ // CHECK: taskgraph map overlap errors: 0
+ printf("taskgraph map overlap errors: %d\n", Errors);
+ free(E);
+ free(D);
+ free(C);
+ free(B);
+ free(A);
+ return Errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_data_block.c b/offload/test/offloading/taskgraph_target_data_block.c
new file mode 100644
index 00000000000000..aa52b590fe1292
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_data_block.c
@@ -0,0 +1,85 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// The 'omp target data' *block* construct nested in an 'omp taskgraph'. It is
+// recorded as three nodes: an enter-data action, a task whose region is the
+// body, and an exit-data action. The inner target is part of that task's
+// region, not of the taskgraph's, so it is not recorded but runs afresh on
+// every replay. The encounter-order edges the runtime builds for undeferred
+// nodes are what keeps the three in order and one region's exit ahead of the
+// next region's enter. Before the construct was recorded at all, the mapping
+// was established by inline host mapper calls that a replay never re-runs, so
+// the maps were silently lost from the second encounter onwards.
+//
+// 'present' on the inner target is what makes that loss a failure rather than a
+// coincidence: without it the inner target's own map would allocate a fresh
+// device buffer per replay and still produce the right answer, hiding the
+// missing region. With it, a replay whose enter-data action did not run (or
+// ran after the body) fails the present check outright. Because the region is
+// present, the inner target's map performs no transfer of its own, so the
+// copy-back also comes from the exit-data action alone.
+//
+// Two sibling regions in the one graph, with non-commuting bodies, so that a
+// replay which lets the first region's exit slip past the second region's enter
+// gets the arithmetic wrong rather than merely running out of order.
+//
+// The last RUN also leaves the AMDGPU plugin free to claim the graph. A graph
+// holding the body of a 'target data' is replayed by libomp on the host
+// instead, and has to reach the same answer.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 1024;
+ const int Iters = 3;
+ int *a = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = i;
+
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : a[0 : N])
+ {
+#pragma omp target map(present, tofrom : a[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] + 1;
+ }
+ }
+
+#pragma omp target data map(tofrom : a[0 : N])
+ {
+#pragma omp target map(present, tofrom : a[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] * 2;
+ }
+ }
+ }
+ }
+
+ int errors = 0;
+ for (int i = 0; i < N; ++i) {
+ int expected = i;
+ for (int it = 0; it < Iters; ++it)
+ expected = (expected + 1) * 2;
+ if (a[i] != expected)
+ ++errors;
+ }
+
+ // CHECK: taskgraph target data block errors: 0
+ printf("taskgraph target data block errors: %d\n", errors);
+ free(a);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_data_body.c b/offload/test/offloading/taskgraph_target_data_body.c
new file mode 100644
index 00000000000000..21d743d7c891ce
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_data_body.c
@@ -0,0 +1,98 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env KMP_TASKGRAPH_TRACE=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic \
+// RUN: --check-prefixes=CHECK,TRACE
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// OpenMP 6.1 [21.5]: the structured block of a 'target data' is a task of its
+// own, ordered between the enter-data and exit-data tasks. A taskgraph
+// records exactly those three. The body task's region is not part of the
+// taskgraph region, so what it encounters is not recorded but runs afresh on
+// every replay, and taskgraph restrictions (a taskwait needing 'depend', for
+// one) do not apply inside it.
+//
+// The body runs its region in a taskgroup, so the exit-data task that copies
+// the data back waits for nested work the body left running: here a nested
+// task that is still writing when the body's own code has finished.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 256
+#define ITERS 4
+
+int main() {
+ int *A = (int *)malloc(N * sizeof(int));
+ for (int I = 0; I < N; ++I)
+ A[I] = 0;
+ int BodyRuns = 0, NestedRuns = 0, AfterWait = 0, BadWait = 0;
+
+ for (int It = 0; It < ITERS; ++It) {
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : A[0 : N])
+ {
+ ++BodyRuns;
+ int Seen = -1;
+#pragma omp task shared(NestedRuns, Seen)
+ {
+ ++NestedRuns;
+ Seen = It;
+ }
+#pragma omp taskwait
+ if (Seen != It)
+ ++BadWait;
+ ++AfterWait;
+
+ // Left running when the body's code ends; only the body's taskgroup
+ // keeps the exit-data task from copying A back before it is done.
+#pragma omp task
+ {
+#pragma omp target map(present, tofrom : A[0 : N])
+ for (int I = 0; I < N; ++I)
+ A[I] += 1;
+ }
+ }
+ }
+ }
+
+ int Errors = 0;
+ for (int I = 0; I < N; ++I)
+ if (A[I] != ITERS)
+ ++Errors;
+
+ // The recorded graph holds the three parts of the construct and nothing
+ // from inside the body.
+ // TRACE: Processed taskgraph
+ // TRACE-NEXT: sequential {
+ // TRACE-NEXT: target enter data:
+ // TRACE-NEXT: node:
+ // TRACE-NEXT: target exit data:
+ // TRACE-NEXT: }
+ // The enter-data and exit-data each go to the device as a subgraph of their
+ // own, and libomp runs the body between them. A is mapped by nothing
+ // outside the graph, so the nested region's 'present' finds it only because
+ // the enter-data's subgraph leaves its mapping in place for the exit-data's
+ // to remove, as the two constructs would outside a graph.
+ // TRACE: Transmitted taskgraph {{.*}} subgraph
+ // TRACE: Transmitted taskgraph {{.*}} subgraph
+ // TRACE-NOT: replays on the host
+
+ // CHECK: body runs: 4
+ printf("body runs: %d\n", BodyRuns);
+ // CHECK: nested task runs: 4
+ printf("nested task runs: %d\n", NestedRuns);
+ // CHECK: past taskwait: 4, saw stale value: 0
+ printf("past taskwait: %d, saw stale value: %d\n", AfterWait, BadWait);
+ // CHECK: copy-back errors: 0
+ printf("copy-back errors: %d\n", Errors);
+ free(A);
+ return Errors != 0 || BadWait != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_data_body_visible.c b/offload/test/offloading/taskgraph_target_data_body_visible.c
new file mode 100644
index 00000000000000..523a850e7eb26a
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_data_body_visible.c
@@ -0,0 +1,100 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env KMP_TASKGRAPH_TRACE=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic \
+// RUN: --check-prefixes=CHECK,GRAPH \
+// RUN: --implicit-check-not='replays on the host'
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// A graph holding the body of a 'target data' is replayed with the target
+// constructs either side of the body on the device, as subgraphs, and the body
+// run by libomp between them. The body is code the graph cannot see, and it
+// finds the construct's data the way any code does: through the mapping table.
+// So what the subgraphs before it map, and leave for those after it to unmap,
+// is published there rather than kept as a subgraph's private storage.
+//
+// Nothing outside the graph maps A, B or C, so every mapping the body finds is
+// one the graph made:
+// - A is the construct's own data: the use_device_ptr conversion,
+// omp_get_mapped_ptr and a nested 'present' region must all find it, and
+// agree on where it is.
+// - B is mapped by a 'target enter data' before the construct and unmapped
+// by a 'target exit data' after it, so it is present in the body too.
+// - C is mapped only by a 'target' region that has finished before the body
+// starts, so it is not present there, and stays the graph's own.
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 64
+#define ITERS 4
+
+int main() {
+ int Dev = omp_get_default_device();
+ int *A = (int *)malloc(N * sizeof(int));
+ int *B = (int *)malloc(N * sizeof(int));
+ int *C = (int *)malloc(N * sizeof(int));
+ int BadUdp = 0, BadA = 0, BadB = 0, BadC = 0, BadData = 0;
+
+ // Within the body A names the device copy, so the host array is looked up
+ // through another name.
+ int *HostA = A;
+
+ for (int It = 0; It < ITERS; ++It) {
+ for (int I = 0; I < N; ++I)
+ A[I] = B[I] = C[I] = 0;
+
+#pragma omp taskgraph
+ {
+#pragma omp target map(tofrom : C[0 : N])
+ for (int I = 0; I < N; ++I)
+ C[I] = 1;
+#pragma omp target enter data map(to : B[0 : N])
+#pragma omp target data map(tofrom : A[0 : N]) use_device_ptr(A)
+ {
+ void *MappedA = omp_get_mapped_ptr(HostA, Dev);
+ if (!MappedA || A != MappedA)
+ ++BadUdp;
+#pragma omp target is_device_ptr(A)
+ for (int I = 0; I < N; ++I)
+ A[I] = It + 1;
+ if (!omp_target_is_present(B, Dev) || !omp_get_mapped_ptr(B, Dev))
+ ++BadB;
+ if (omp_target_is_present(C, Dev))
+ ++BadC;
+#pragma omp target map(present, tofrom : B[0 : N])
+ for (int I = 0; I < N; ++I)
+ B[I] = It + 2;
+ }
+#pragma omp target exit data map(from : B[0 : N])
+ }
+
+ for (int I = 0; I < N; ++I)
+ if (A[I] != It + 1 || B[I] != It + 2 || C[I] != 1)
+ ++BadData;
+ if (omp_target_is_present(A, Dev) || omp_target_is_present(B, Dev))
+ ++BadA;
+ }
+
+ // GRAPH: Transmitted taskgraph
+ // CHECK: use_device_ptr mismatches: 0
+ printf("use_device_ptr mismatches: %d\n", BadUdp);
+ // CHECK: B absent in body: 0
+ printf("B absent in body: %d\n", BadB);
+ // CHECK: C present in body: 0
+ printf("C present in body: %d\n", BadC);
+ // CHECK: left mapped after replay: 0
+ printf("left mapped after replay: %d\n", BadA);
+ // CHECK: data errors: 0
+ printf("data errors: %d\n", BadData);
+ free(A);
+ free(B);
+ free(C);
+ return (BadUdp + BadA + BadB + BadC + BadData) != 0;
+}
diff --git a/offload/test/offloading/taskgraph_use_device_ptr.c b/offload/test/offloading/taskgraph_use_device_ptr.c
new file mode 100644
index 00000000000000..8837233fab60f0
--- /dev/null
+++ b/offload/test/offloading/taskgraph_use_device_ptr.c
@@ -0,0 +1,93 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env KMP_TASKGRAPH_TRACE=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic \
+// RUN: --check-prefixes=CHECK,GRAPH \
+// RUN: --implicit-check-not='replays on the host'
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
+// first to do.
+// XFAIL: *
+
+// The body of a recorded 'target data' is a task, which converts the
+// use_device_ptr list item itself whenever it starts. So each replay sees the
+// device address the list item has now, not the one it had when the graph was
+// recorded: the enclosing mapping is made afresh around every replay, behind a
+// growing allocation, so that it is unlikely to land in the same place twice.
+//
+// Within the body the list item is an ordinary variable holding the device
+// address. A nested task finds it there under the default data-sharing rules
+// (firstprivate) and when it is shared, and a nested target can hand it to
+// the kernel with is_device_ptr. The nested constructs are part of the body
+// task's region, not the taskgraph's, so they are not recorded but run afresh
+// on each replay.
+//
+// The construct's enter-data and exit-data still go to the device plugin, as
+// subgraphs either side of the body, which libomp runs.
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 64
+#define ITERS 4
+
+int main() {
+ int Dev = omp_get_default_device();
+ int *A = (int *)malloc(N * sizeof(int));
+ int *Pad[ITERS];
+ int BadFp = 0, BadSh = 0, BadTgt = 0;
+
+ for (int It = 0; It < ITERS; ++It) {
+ for (int I = 0; I < N; ++I)
+ A[I] = -1;
+ // Something new on the device ahead of A each time, so that A's device
+ // copy moves.
+ Pad[It] = (int *)malloc((It + 1) * 4096);
+#pragma omp target enter data map(alloc : Pad[It][0 : (It + 1) * 1024])
+#pragma omp target enter data map(to : A[0 : N])
+ void *WantA = omp_get_mapped_ptr(A, Dev);
+ void *SeenFp = NULL, *SeenSh = NULL;
+
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : A[0 : N]) use_device_ptr(A)
+ {
+#pragma omp task shared(SeenFp)
+ SeenFp = A;
+#pragma omp task shared(A, SeenSh)
+ SeenSh = A;
+#pragma omp target is_device_ptr(A)
+ for (int I = 0; I < N; ++I)
+ A[I] = It;
+ }
+ }
+
+#pragma omp target exit data map(from : A[0 : N])
+ if (SeenFp != WantA)
+ ++BadFp;
+ if (SeenSh != WantA)
+ ++BadSh;
+ if (A[0] != It || A[N - 1] != It)
+ ++BadTgt;
+ }
+ for (int It = 0; It < ITERS; ++It) {
+#pragma omp target exit data map(release : Pad[It][0 : (It + 1) * 1024])
+ free(Pad[It]);
+ }
+
+ // GRAPH: Transmitted taskgraph
+ // CHECK: firstprivate task mismatches: 0
+ printf("firstprivate task mismatches: %d\n", BadFp);
+ // CHECK: shared task mismatches: 0
+ printf("shared task mismatches: %d\n", BadSh);
+ // CHECK: is_device_ptr target mismatches: 0
+ printf("is_device_ptr target mismatches: %d\n", BadTgt);
+ free(A);
+ return (BadFp + BadSh + BadTgt) != 0;
+}
diff --git a/offload/test/offloading/taskgraph_use_device_ptr_fallback.c b/offload/test/offloading/taskgraph_use_device_ptr_fallback.c
new file mode 100644
index 00000000000000..b2a615a081d397
--- /dev/null
+++ b/offload/test/offloading/taskgraph_use_device_ptr_fallback.c
@@ -0,0 +1,63 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=61 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compile-generic -fopenmp-version=61 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+
+// A `use_device_ptr` list item that nothing maps has no device address to be
+// told, and the clause says what to hand back instead: the host address by
+// default, or null under `fb_nullify`. Inside a recorded `target data` the
+// body is a task that asks for the answer afresh whenever it starts, so the
+// fallback has to be taken there too, on every replay. An absent entry is
+// found by asking the mapping table, which answers with the host address
+// rather than with a null, so a fallback keyed on a null answer is never
+// taken.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define N 64
+#define ITERS 3
+
+int main() {
+ int *A = (int *)malloc(N * sizeof(int));
+ int *Kept = (int *)malloc(N * sizeof(int));
+ int *Nulled = (int *)malloc(N * sizeof(int));
+ int WrongKept = 0, WrongNulled = 0;
+
+ for (int It = 0; It < ITERS; ++It) {
+ int *SeenKept = NULL, *SeenNulled = Nulled;
+
+#pragma omp taskgraph
+ {
+#pragma omp target data map(tofrom : A[0 : N]) use_device_ptr(Kept) \
+ use_device_ptr(fb_nullify : Nulled)
+ {
+#pragma omp task shared(Kept, Nulled, SeenKept, SeenNulled)
+ {
+ SeenKept = Kept;
+ SeenNulled = Nulled;
+ }
+ }
+ }
+
+ if (SeenKept != Kept)
+ ++WrongKept;
+ if (SeenNulled != NULL)
+ ++WrongNulled;
+ }
+
+ // CHECK: replays not handed the host address: 0
+ printf("replays not handed the host address: %d\n", WrongKept);
+ // CHECK: replays not handed null: 0
+ printf("replays not handed null: %d\n", WrongNulled);
+
+ free(Nulled);
+ free(Kept);
+ free(A);
+ return (WrongKept + WrongNulled) != 0;
+}
>From 3480e576f746d453b40d6d4ff007ee2fe3d6cc96 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 1 Oct 2026 07:00:40 -0500
Subject: [PATCH 24/29] [Offload] Cache taskgraph kernel launch parameters
The shape of a kernel launch is fixed from one replay to the next,
so we can calculate launch parameters once and then cache them for
subsequent replays. This is done via reuse of the the helpers that
DeviceTy::launchKernel uses to calculate the geometry of launches for
the non-taskgraph case.
Assisted-By: Claude
---
offload/include/Shared/TaskGraph.h | 20 ++++
offload/include/device.h | 14 +++
offload/libompaccsupport/device.cpp | 87 +++++++++++++++
offload/libomptarget/OpenMP/TaskGraphPlan.cpp | 36 ++++++
.../offloading/taskgraph_target_launch_env.c | 103 ++++++++++++++++++
5 files changed, 260 insertions(+)
create mode 100644 offload/test/offloading/taskgraph_target_launch_env.c
diff --git a/offload/include/Shared/TaskGraph.h b/offload/include/Shared/TaskGraph.h
index ffc4697ab01b93..c0a9c6cba47099 100644
--- a/offload/include/Shared/TaskGraph.h
+++ b/offload/include/Shared/TaskGraph.h
@@ -378,6 +378,19 @@ struct TaskGraphNode : TaskGraphElement {
uint32_t NumOps = 0;
};
+/// A target leaf's kernel launch parameters: what a launch outside a graph
+/// works out every time, calculated once when libomptarget builds the plan and
+/// cached for every replay. The device memory behind them belongs to the plan.
+struct TaskGraphLaunchTy {
+ uint32_t NumThreads[3] = {1, 1, 1};
+ uint32_t NumBlocks[3] = {1, 1, 1};
+ /// Dynamic group memory provided natively, on top of the kernel's static
+ /// allocation.
+ uint32_t DynBlockMemSize = 0;
+ /// The value of the kernel's launch-environment (dyn_ptr) argument.
+ void *LaunchEnvironment = nullptr;
+};
+
/// A target kernel launch. KernelArgs is an opaque KernelArgsTy*, deep-copied
/// by libomp at record time.
struct TaskGraphTargetNode : TaskGraphNode {
@@ -387,6 +400,9 @@ struct TaskGraphTargetNode : TaskGraphNode {
void *KernelArgs = nullptr;
/// The kernel's explicit parameters, in the kernel's order.
inline llvm::ArrayRef<TaskGraphKernelParamTy> kernelParams() const;
+ /// The geometry, dynamic group memory and launch environment every replay
+ /// launches the kernel with.
+ const TaskGraphLaunchTy &launch() const { return Launch; }
TaskGraphTargetNode() : TaskGraphNode(TGKind::TargetNode) {}
static bool classof(const TaskGraphElement *E) {
@@ -399,6 +415,7 @@ struct TaskGraphTargetNode : TaskGraphNode {
// Set when the plan is built.
TaskGraphKernelParamTy *Params = nullptr;
uint32_t NumParams = 0;
+ TaskGraphLaunchTy Launch;
};
/// Base for target enter/exit/update data ops. The specific op is encoded by
@@ -776,6 +793,9 @@ class TaskGraphDataPlanTy {
std::vector<TaskGraphOpTy> Ops;
/// Every target leaf's kernel parameters, end to end.
std::vector<TaskGraphKernelParamTy> KernelParams;
+ /// Device memory the cached launches own: launch environments, reduction
+ /// buffers and global-memory fallbacks for dynamic group memory.
+ std::vector<void *> LaunchAllocations;
/// The mapping-table epoch as of the plan's build or its last release of the
/// mappings. Only used to report that the program changed the mapping state
diff --git a/offload/include/device.h b/offload/include/device.h
index f784219df4c1ea..d503d3cc0dcdea 100644
--- a/offload/include/device.h
+++ b/offload/include/device.h
@@ -108,6 +108,9 @@ struct DeviceGeometryPropsTy {
};
struct PluginManager;
+namespace llvm::omp::target {
+struct TaskGraphLaunchTy;
+} // namespace llvm::omp::target
struct DeviceTy {
int32_t DeviceID;
@@ -233,6 +236,17 @@ struct DeviceTy {
KernelReplayOutcomeTy *ReplayOutcome,
AsyncInfoTy &AsyncInfo);
+ /// Calculate, once, the launch parameters of the kernel \p TgtEntryPtr with
+ /// \p KernelArgs for a taskgraph that replays it, and cache them in
+ /// \p Launch: the geometry, the dynamic group memory and the launch
+ /// environment launchKernel works out on every launch. The device memory it
+ /// allocates is appended to \p Allocations, for the caller to free with
+ /// freeCtxMem.
+ llvm::Error cacheTaskGraphLaunch(void *TgtEntryPtr,
+ const KernelArgsTy &KernelArgs,
+ llvm::omp::target::TaskGraphLaunchTy &Launch,
+ llvm::SmallVectorImpl<void *> &Allocations);
+
/// Synchronize device/queue/event based on \p AsyncInfo and return
/// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails.
int32_t synchronize(AsyncInfoTy &AsyncInfo);
diff --git a/offload/libompaccsupport/device.cpp b/offload/libompaccsupport/device.cpp
index 4934c940d02be7..b21a7e3c38e48e 100644
--- a/offload/libompaccsupport/device.cpp
+++ b/offload/libompaccsupport/device.cpp
@@ -19,6 +19,7 @@
#include "PluginManager.h"
#include "Shared/APITypes.h"
#include "Shared/Debug.h"
+#include "Shared/TaskGraph.h"
#include "omptarget.h"
#include "rtl.h"
@@ -1129,6 +1130,92 @@ prepareDynamicLaunchState(DeviceTy &Device, uint32_t OmpABIVersion,
return llvm::Error::success();
}
+llvm::Error
+DeviceTy::cacheTaskGraphLaunch(void *TgtEntryPtr,
+ const KernelArgsTy &KernelArgs,
+ llvm::omp::target::TaskGraphLaunchTy &Launch,
+ llvm::SmallVectorImpl<void *> &Allocations) {
+ GenericDeviceTy &GenericDevice = RTL->getDevice(RTLDeviceID);
+ KernelLaunchInfoTy KernelLaunchInfo = getKernelLaunchInfo(TgtEntryPtr);
+ KernelLaunchArgsTy LaunchArgs = buildLaunchArgs(KernelArgs);
+ checkLaunchInvariants(LaunchArgs, KernelArgs, KernelLaunchInfo);
+ adjustEffectiveGeometry(GenericDevice, GeometryProps, LaunchArgs, KernelArgs,
+ KernelLaunchInfo);
+ llvm::copy(LaunchArgs.UserThreadLimit, Launch.NumThreads);
+ llvm::copy(LaunchArgs.UserNumBlocks, Launch.NumBlocks);
+
+ // The geometry is fixed for the graph, so the fallback for dynamic group
+ // memory can be allocated once rather than per launch.
+ auto DynBlockMemConfOrErr =
+ prepareBlockMemory(*this, KernelLaunchInfo, KernelArgs.DynCGroupMem,
+ static_cast<DynCGroupMemFallbackType>(
+ KernelArgs.Flags.DynCGroupMemFallback),
+ uint64_t(Launch.NumBlocks[0]) * Launch.NumBlocks[1] *
+ Launch.NumBlocks[2]);
+ if (!DynBlockMemConfOrErr)
+ return DynBlockMemConfOrErr.takeError();
+ const DynBlockMemConfTy &DynBlockMemConf = *DynBlockMemConfOrErr;
+ if (DynBlockMemConf.FallbackPtr)
+ Allocations.push_back(DynBlockMemConf.FallbackPtr);
+ Launch.DynBlockMemSize = DynBlockMemConf.NativeSize;
+
+ // A backend launches the cached kernel itself, so the limits the plugin
+ // checks on every launch are checked here instead.
+ const uint64_t MaxBlockMemSize = getMaxBlockSharedMemSize();
+ if (KernelLaunchInfo.StaticBlockMemSize > MaxBlockMemSize)
+ return createError(ErrorCode::InvalidValue,
+ "static block memory size exceeds maximum");
+ if (KernelLaunchInfo.StaticBlockMemSize + Launch.DynBlockMemSize >
+ MaxBlockMemSize)
+ return createError(
+ ErrorCode::InvalidValue,
+ "requested block memory size (static + dynamic) exceeds maximum");
+
+ // As in getKernelLaunchEnvironment, except that the environment is the
+ // graph's, written once, and reused by every replay. A replay runs at most
+ // one launch of a leaf at a time, and the reduction's teams-done counter
+ // resets itself, so nothing in it has to be reinitialised between launches.
+ const bool NeedsReductionBuffer = KernelLaunchInfo.ReductionDataSize != 0;
+ if (NeedsReductionBuffer && KernelArgs.Version < OMP_KERNEL_ARG_VERSION)
+ return createError(
+ ErrorCode::InvalidBinary,
+ "kernel was built against an older OpenMP kernel-launch-environment "
+ "ABI (v%u); current runtime requires v%u for cross-team reductions",
+ KernelArgs.Version, OMP_KERNEL_ARG_VERSION);
+ if (!NeedsReductionBuffer && !KernelArgs.DynCGroupMem) {
+ Launch.LaunchEnvironment = reinterpret_cast<void *>(~uintptr_t(0));
+ return llvm::Error::success();
+ }
+
+ KernelLaunchEnvironmentTy KLE{};
+ KLE.DynCGroupMemSize = DynBlockMemConf.Size;
+ KLE.DynCGroupMemFbPtr = DynBlockMemConf.FallbackPtr;
+ KLE.DynCGroupMemFb = DynBlockMemConf.Fallback;
+ KLE.ReductionBuffer = nullptr;
+ if (NeedsReductionBuffer) {
+ auto AllocOrErr = allocCtxMem(uint64_t(KernelLaunchInfo.ReductionDataSize) *
+ Launch.NumBlocks[0],
+ OL_ALLOC_TYPE_DEVICE);
+ if (!AllocOrErr)
+ return AllocOrErr.takeError();
+ Allocations.push_back(*AllocOrErr);
+ KLE.ReductionBuffer = *AllocOrErr;
+ }
+
+ auto AllocOrErr =
+ allocCtxMem(sizeof(KernelLaunchEnvironmentTy), OL_ALLOC_TYPE_DEVICE);
+ if (!AllocOrErr)
+ return AllocOrErr.takeError();
+ Allocations.push_back(*AllocOrErr);
+ if (submitDataSync(*AllocOrErr, &KLE, sizeof(KLE), /*Entry=*/nullptr,
+ /*HDTTMapPtr=*/nullptr, /*Loc=*/nullptr,
+ /*Name=*/"KernelLaunchEnv") != OFFLOAD_SUCCESS)
+ return createError(ErrorCode::BackendFailure,
+ "failed to copy the kernel launch environment");
+ Launch.LaunchEnvironment = *AllocOrErr;
+ return llvm::Error::success();
+}
+
// Run region on device
int32_t DeviceTy::launchKernel(void *TgtEntryPtr, void **TgtVarsPtr,
ptrdiff_t *TgtOffsets, KernelArgsTy &KernelArgs,
diff --git a/offload/libomptarget/OpenMP/TaskGraphPlan.cpp b/offload/libomptarget/OpenMP/TaskGraphPlan.cpp
index f387edc4502143..da3ddcafa9f4be 100644
--- a/offload/libomptarget/OpenMP/TaskGraphPlan.cpp
+++ b/offload/libomptarget/OpenMP/TaskGraphPlan.cpp
@@ -27,6 +27,7 @@
using namespace llvm;
using namespace llvm::omp::target;
+using namespace llvm::omp::target::debug;
namespace mt = omptarget::maptype;
@@ -1511,6 +1512,34 @@ bool TaskGraphPlanner::buildTaskGraphPlan(DeviceTy &Device,
return false;
}
}
+
+ // A backend launches each kernel many times from one lowering, so what a
+ // launch outside a graph works out every time -- the geometry, the dynamic
+ // group memory and the launch environment -- is calculated once, here, and
+ // cached. A leaf whose launch cannot be calculated leaves the graph to the
+ // host, as a backend meeting something it cannot express does.
+ for (TaskGraphNode *Node : Graph.leaves()) {
+ auto *Leaf = dyn_cast<TaskGraphTargetNode>(Node);
+ if (!Leaf)
+ continue;
+ void *Kernel = Graph.Services.resolveKernel(Graph.DeviceId, Leaf->HostPtr);
+ if (!Kernel) {
+ ODBG(ODT_Device) << "device " << Device.DeviceID << " declined taskgraph "
+ << &Graph
+ << ": target leaf's kernel is not loaded on this device";
+ return false;
+ }
+ SmallVector<void *, 3> Allocations;
+ Error Err = Device.cacheTaskGraphLaunch(
+ Kernel, *static_cast<const KernelArgsTy *>(Leaf->KernelArgs),
+ Leaf->Launch, Allocations);
+ llvm::append_range(Plan.LaunchAllocations, Allocations);
+ if (Err) {
+ ODBG(ODT_Device) << "device " << Device.DeviceID << " declined taskgraph "
+ << &Graph << ": " << toString(std::move(Err));
+ return false;
+ }
+ }
return true;
}
@@ -1717,6 +1746,13 @@ void TaskGraphPlanner::releaseTaskGraphPlan(DeviceTy &Device,
"privatised arguments";
Graph.Plan.PrivateBlock = nullptr;
}
+
+ for (void *Ptr : Graph.Plan.LaunchAllocations)
+ if (Error Err = Device.freeCtxMem(Ptr))
+ REPORT() << "taskgraph could not release the device memory of a cached "
+ "kernel launch: "
+ << toString(std::move(Err));
+ Graph.Plan.LaunchAllocations.clear();
}
bool llvm::omp::target::buildTaskGraphPlan(DeviceTy &Device,
diff --git a/offload/test/offloading/taskgraph_target_launch_env.c b/offload/test/offloading/taskgraph_target_launch_env.c
new file mode 100644
index 00000000000000..efeac6da904eae
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_launch_env.c
@@ -0,0 +1,103 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=61 && \
+// RUN: env LIBOMPTARGET_DEBUG=1 %libomptarget-run-generic 2>&1 | \
+// RUN: %fcheck-generic --implicit-check-not='declined taskgraph'
+
+// REQUIRES: gpu
+// REQUIRES: amdgpu
+// REQUIRES: omp_taskgraph_experimental
+// REQUIRES: libomptarget-debug
+
+// A kernel launch outside a graph gets a launch environment of its own when it
+// needs one: for a cross-team reduction's buffer, and for dynamic group memory
+// and its fallback. A device graph has to provide the same, once, for every
+// replay; without it the reduction dereferences a null buffer and the dynamic
+// group memory comes back empty.
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 256
+#define TEAMS 64
+#define ITERS 4
+
+int main() {
+ int Errors = 0;
+
+ // A cross-team reduction, which needs a reduction buffer.
+ for (int It = 0; It < ITERS; ++It) {
+ long Sum = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target teams distribute parallel for num_teams(TEAMS) \
+ reduction(+ : Sum) map(tofrom : Sum)
+ for (int I = 0; I < TEAMS * N; ++I)
+ Sum += I;
+ }
+ if (Sum != (long)TEAMS * N * (TEAMS * N - 1) / 2) {
+ printf("reduction replay %d: sum %ld\n", It, Sum);
+ ++Errors;
+ }
+ }
+
+ // Dynamic group memory the device provides natively, which the kernel finds
+ // through its launch environment.
+ for (int It = 0; It < ITERS; ++It) {
+ int Result[N];
+ size_t Size = 0;
+#pragma omp taskgraph
+ {
+#pragma omp target teams num_teams(1) thread_limit(N) \
+ dyn_groupprivate(fallback(abort) : N * sizeof(int)) \
+ map(from : Result, Size)
+ {
+#pragma omp parallel
+ {
+ int *Dyn = (int *)omp_get_dyn_gprivate_nofb_ptr(0, omp_access_cgroup);
+ int T = omp_get_thread_num();
+ int NT = omp_get_num_threads();
+ if (T == 0)
+ Size = omp_get_dyn_gprivate_size(omp_access_cgroup);
+ if (Dyn)
+ Dyn[T] = T + It;
+#pragma omp barrier
+ Result[T] = Dyn ? Dyn[(T + 1) % NT] : -1;
+ }
+ }
+ }
+ if (Size != N * sizeof(int) || Result[0] != 1 + It) {
+ printf("native replay %d: size %zu, result %d\n", It, Size, Result[0]);
+ ++Errors;
+ }
+ }
+
+ // More dynamic group memory than the device has, which falls back to global
+ // memory, one slice per team.
+ size_t Big = omp_get_gprivate_limit(0, omp_access_cgroup) + sizeof(int);
+ for (int It = 0; It < ITERS; ++It) {
+ int Result[TEAMS];
+#pragma omp taskgraph
+ {
+#pragma omp target teams num_teams(TEAMS) thread_limit(1) \
+ dyn_groupprivate(fallback(default_mem) : Big) map(from : Result)
+ {
+ int *Dyn = (int *)omp_get_dyn_gprivate_ptr(0, omp_access_cgroup);
+ int T = omp_get_team_num();
+ if (Dyn) {
+ Dyn[0] = T + It;
+ Dyn[Big / sizeof(int) - 1] = T;
+ }
+ Result[T] = Dyn ? Dyn[0] + Dyn[Big / sizeof(int) - 1] : -1;
+ }
+ }
+ for (int T = 0; T < TEAMS; ++T)
+ if (Result[T] != 2 * T + It) {
+ printf("fallback replay %d: team %d result %d\n", It, T, Result[T]);
+ ++Errors;
+ break;
+ }
+ }
+
+ // CHECK: taskgraph launch environment errors: 0
+ printf("taskgraph launch environment errors: %d\n", Errors);
+ return Errors != 0;
+}
>From a06a5310978db557be502e77569cc7f1f602ed4c Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 19 Aug 2026 11:17:56 -0500
Subject: [PATCH 25/29] [Offload][AMDGPU] Let streams wait on a set of events
This is an addition to the AMDGPU offload plugin for the use of the
taskgraph backend.
The barrier-packet helper currently only supports two named signals.
This patch raises that to the HSA API's limit, which is five.
The new waitEvents function makes one stream wait on several events at
once, via the above extended helper.
Assisted-By: Claude
---
offload/plugins-nextgen/amdgpu/src/rtl.cpp | 128 +++++++++++++++++++--
1 file changed, 117 insertions(+), 11 deletions(-)
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 28a8a8e45fe494..05de52075b491a 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -10,6 +10,7 @@
//
//===----------------------------------------------------------------------===//
+#include <algorithm>
#include <atomic>
#include <cassert>
#include <cstddef>
@@ -817,6 +818,9 @@ struct AMDGPUQueueTy {
return Plugin::check(Status, "error in hsa_queue_destroy: %s");
}
+ /// Maximum number of dependency signals a single barrier-AND packet carries.
+ static constexpr unsigned MaxBarrierDeps = 5;
+
/// Returns the number of streams, this queue is currently assigned to.
bool getUserCount() const { return NumUsers; }
@@ -898,14 +902,35 @@ struct AMDGPUQueueTy {
return pushBarrierImpl(OutputSignal, InputSignal1, InputSignal2);
}
+ /// Push a barrier packet waiting on up to MaxBarrierDeps input signals. Null
+ /// entries in \p InputSignals are skipped, so a caller can pass an optional
+ /// signal (e.g. a stream's own chain signal) without filtering it first.
+ Error pushBarrierDeps(AMDGPUSignalTy *OutputSignal,
+ ArrayRef<const AMDGPUSignalTy *> InputSignals) {
+ std::lock_guard<std::mutex> Lock(Mutex);
+ assert(Queue && "Interacted with a non-initialized queue!");
+ return pushBarrierDepsImpl(OutputSignal, InputSignals);
+ }
+
private:
/// Push a barrier packet that will wait up to two input signals. Assumes the
/// the queue lock is acquired.
Error pushBarrierImpl(AMDGPUSignalTy *OutputSignal,
const AMDGPUSignalTy *InputSignal1,
const AMDGPUSignalTy *InputSignal2 = nullptr) {
- // Add a queue barrier waiting on both the other stream's operation and the
- // last operation on the current stream (if any).
+ const AMDGPUSignalTy *Deps[] = {InputSignal1, InputSignal2};
+ return pushBarrierDepsImpl(OutputSignal, Deps);
+ }
+
+ /// Push a barrier packet waiting on every non-null signal in \p InputSignals.
+ /// Assumes the queue lock is acquired.
+ Error pushBarrierDepsImpl(AMDGPUSignalTy *OutputSignal,
+ ArrayRef<const AMDGPUSignalTy *> InputSignals) {
+ assert(InputSignals.size() <= MaxBarrierDeps &&
+ "too many dependencies for one barrier packet");
+
+ // Add a queue barrier waiting on the other streams' operations and the last
+ // operation on the current stream (if any).
uint64_t PacketId;
hsa_barrier_and_packet_t *Packet =
(hsa_barrier_and_packet_t *)acquirePacket(PacketId);
@@ -913,21 +938,18 @@ struct AMDGPUQueueTy {
Packet->reserved0 = 0;
Packet->reserved1 = 0;
- Packet->dep_signal[0] = {0};
- Packet->dep_signal[1] = {0};
- Packet->dep_signal[2] = {0};
- Packet->dep_signal[3] = {0};
- Packet->dep_signal[4] = {0};
+ for (unsigned I = 0; I < MaxBarrierDeps; ++I)
+ Packet->dep_signal[I] = {0};
Packet->reserved2 = 0;
Packet->completion_signal = {0};
// Set input and output dependencies if needed.
if (OutputSignal)
Packet->completion_signal = OutputSignal->get();
- if (InputSignal1)
- Packet->dep_signal[0] = InputSignal1->get();
- if (InputSignal2)
- Packet->dep_signal[1] = InputSignal2->get();
+ unsigned Slot = 0;
+ for (const AMDGPUSignalTy *InputSignal : InputSignals)
+ if (InputSignal)
+ Packet->dep_signal[Slot++] = InputSignal->get();
// Publish the packet. Do not modify the packet after this point.
publishBarrierPacket(PacketId, Packet);
@@ -1746,6 +1768,17 @@ struct AMDGPUStreamTy {
/// Make the stream wait on an event.
Error waitEvent(const AMDGPUEventTy &Event);
+ /// Make the stream wait on a whole set of events at once.
+ ///
+ /// One barrier-AND packet carries several dependency signals, so a set of
+ /// events costs one packet (and one stream slot) instead of one per event as
+ /// repeated waitEvent calls would. Taskgraph replay gates each item on at
+ /// most one event per other lane, which is exactly this shape.
+ ///
+ /// Null entries and events that need no wait (nothing recorded, already
+ /// complete, or recorded on this same stream) are skipped.
+ Error waitEvents(ArrayRef<AMDGPUEventTy *> Events);
+
private:
/// Lightweight sync-only recording.
Error recordEventSyncOnly(AMDGPUEventTy &Event, AMDGPUSignalTy *ReusedSignal);
@@ -1962,6 +1995,79 @@ Error AMDGPUStreamTy::waitEvent(const AMDGPUEventTy &Event) {
return waitOnStreamOperation(RecordedStream, Event.RecordedSlot);
}
+Error AMDGPUStreamTy::waitEvents(ArrayRef<AMDGPUEventTy *> Events) {
+ if (Queue == nullptr)
+ return Plugin::error(ErrorCode::INVALID_NULL_POINTER,
+ "target queue was nullptr");
+
+ // Gather the dependencies that are still outstanding. Each producing
+ // stream's lock is taken and dropped in turn, so two stream locks are never
+ // held together and this cannot deadlock against a concurrent waitEvent.
+ // Every gathered signal has its use count raised so its producing stream
+ // cannot recycle it before the barrier below is published; the matching
+ // release is scheduled on the slot the barrier consumes.
+ SmallVector<AMDGPUSignalTy *, AMDGPUQueueTy::MaxBarrierDeps> Deps;
+ for (AMDGPUEventTy *Event : Events) {
+ if (!Event)
+ continue;
+ std::lock_guard<std::mutex> EventLock(Event->Mutex);
+ AMDGPUStreamTy *Recorded = Event->RecordedStream;
+ // Nothing recorded, or recorded on this very stream, where in-order
+ // execution already provides the ordering.
+ if (!Recorded || Recorded == this || Event->RecordedSlot < 0)
+ continue;
+
+ std::lock_guard<std::mutex> RecordedLock(Recorded->Mutex);
+ // A later synchronize already finalized the recorded operation.
+ if (Recorded->SyncCycle != (uint32_t)Event->RecordedSyncCycle)
+ continue;
+ AMDGPUSignalTy *Signal = Recorded->Slots[Event->RecordedSlot].Signal;
+ if (!Signal || !Signal->load())
+ continue; // the recorded operation has already completed.
+
+ Signal->increaseUseCount();
+ Deps.push_back(Signal);
+ }
+
+ if (Deps.empty())
+ return Plugin::success();
+
+ // One dependency slot of each packet goes to this stream's own chain signal,
+ // leaving the rest for cross-stream dependencies; a larger set is split over
+ // consecutive packets. Splitting is sound because the packets land in order
+ // on one queue, so anything pushed afterwards is ordered after all of them.
+ const size_t DepsPerPacket = AMDGPUQueueTy::MaxBarrierDeps - 1;
+ for (size_t First = 0; First < Deps.size(); First += DepsPerPacket) {
+ ArrayRef<AMDGPUSignalTy *> Chunk = ArrayRef<AMDGPUSignalTy *>(Deps).slice(
+ First, std::min(DepsPerPacket, Deps.size() - First));
+
+ // Retrieve an available signal for the barrier's output.
+ AMDGPUSignalTy *OutputSignal = nullptr;
+ if (auto Err = SignalManager.getResource(OutputSignal))
+ return Err;
+ OutputSignal->reset();
+ OutputSignal->increaseUseCount();
+
+ std::lock_guard<std::mutex> Lock(Mutex);
+
+ // Consume stream slot and compute dependencies.
+ auto [Curr, InputSignal] = consume(OutputSignal);
+
+ // Give back the borrowed signals once this barrier has completed.
+ for (AMDGPUSignalTy *Dep : Chunk)
+ if (auto Err = Slots[Curr].schedReleaseSignal(Dep, &SignalManager))
+ return Err;
+
+ SmallVector<const AMDGPUSignalTy *, AMDGPUQueueTy::MaxBarrierDeps> In;
+ In.push_back(InputSignal);
+ In.append(Chunk.begin(), Chunk.end());
+ if (auto Err = Queue->pushBarrierDeps(OutputSignal, In))
+ return Err;
+ }
+
+ return Plugin::success();
+}
+
Error AMDGPUStreamTy::synchronizeOn(AMDGPUEventTy &Event) {
std::lock_guard<std::mutex> Lock(Mutex);
>From a79534e085cb0b5e32ef9962bffde646d210def9 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 18 Sep 2026 06:43:44 -0500
Subject: [PATCH 26/29] [Offload][AMDGPU] Assign a set of streams to distinct
queues
For the taskgraph implementation, we want to distribute offloaded
kernels over a set of HSA queues, via the plugin's stream abstraction.
The offload plugin uses a least-used heuristic to assign streams to
queues, which cannot be used if several streams are initialised in one
go on an otherwise-idle machine.
This patch adds a mechanism for assigning distinct queues explicitly.
Assisted-By: Claude
---
offload/plugins-nextgen/amdgpu/src/rtl.cpp | 37 ++++++++++++++++++++++
1 file changed, 37 insertions(+)
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 05de52075b491a..07b060a8f203d4 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2149,7 +2149,44 @@ struct AMDGPUStreamManagerTy final
});
}
+ /// Get up to \p Num streams bound to pairwise-distinct HSA queues, appended
+ /// to \p Streams.
+ ///
+ /// This works differently from getResource: its least-used-queue heuristic
+ /// looks at the user counts as they stand at the moment of the request, so a
+ /// batch of streams requested back to back can all be handed the same queue.
+ /// Taskgraph replay needs distinct queues for two reasons: to achieve
+ /// concurrency and to avoid deadlocks which can happen if two streams share
+ /// the same queue.
+ ///
+ /// Fewer than \p Num streams are produced when the device has fewer queues.
+ /// The caller must use the number it actually gets, since that is the lane
+ /// count the schedule has to be built for.
+ Error getDistinctQueueResources(uint32_t Num,
+ SmallVectorImpl<AMDGPUStreamTy *> &Streams) {
+ Num = std::min(Num, MaxNumQueues);
+ for (uint32_t I = 0; I < Num; ++I) {
+ AMDGPUStreamTy *Handle = nullptr;
+ if (auto Err = getResourcesImpl(1, &Handle, [&](AMDGPUStreamTy *&Stream) {
+ return assignQueue(Stream, I);
+ }))
+ return Err;
+ Streams.push_back(Handle);
+ }
+ return Plugin::success();
+ }
+
private:
+ /// Bind \p Stream to queue \p Index, initializing that queue if it is the
+ /// first use.
+ Error assignQueue(AMDGPUStreamTy *Stream, uint32_t Index) {
+ if (auto Err = Queues[Index].init(Device, Agent, QueueSize))
+ return Err;
+ Queues[Index].addUser();
+ Stream->Queue = &Queues[Index];
+ return Plugin::success();
+ }
+
/// Search for and assign an preferably idle queue to the given Stream. If
/// there is no queue without current users, choose the queue with the lowest
/// user count. If utilization is ignored: use round robin selection.
>From ea8e3aced20c3d3a28d50bb419a981adc9d65d05 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 19 Aug 2026 11:20:15 -0500
Subject: [PATCH 27/29] [Offload][AMDGPU] Lower a taskgraph region tree onto
in-order lanes
This is the first part of taskgraph execution support for the AMDGPU
plugin. It implements an algorithm to take the taskgraph representation
held by libomptarget (previously received from libomp) and distribute its
work over a set of in-order lanes. The scheduling is static, and somewhat
independent of the underlying lane representation: for AMDGPU (in the
next patch) lanes are implemented by the plugin's stream implementation.
Since the AMDGPU plugin is the only consumer of this code for the time
being, it's placed in the same directory as the plugin. The lowering
hands each leaf it places, with its lane and the items it has to wait
for, to a TaskGraphLaneSinkTy; the next patch implements that interface
to assign AMDGPU streams and issue the cross-stream waits.
A unit test is added that can test the lane-assignment algorithm
separately from the plugin.
Assisted-By: Claude
---
offload/plugins-nextgen/amdgpu/CMakeLists.txt | 2 +-
.../amdgpu/src/TaskGraphSchedule.cpp | 353 ++++++++
.../amdgpu/src/TaskGraphSchedule.h | 83 ++
offload/unittests/CMakeLists.txt | 5 +
offload/unittests/TaskGraph/CMakeLists.txt | 23 +
.../TaskGraph/TaskGraphScheduleTest.cpp | 829 ++++++++++++++++++
6 files changed, 1294 insertions(+), 1 deletion(-)
create mode 100644 offload/plugins-nextgen/amdgpu/src/TaskGraphSchedule.cpp
create mode 100644 offload/plugins-nextgen/amdgpu/src/TaskGraphSchedule.h
create mode 100644 offload/unittests/TaskGraph/CMakeLists.txt
create mode 100644 offload/unittests/TaskGraph/TaskGraphScheduleTest.cpp
diff --git a/offload/plugins-nextgen/amdgpu/CMakeLists.txt b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
index 4f680087e3fcf3..8c682bb3207a1d 100644
--- a/offload/plugins-nextgen/amdgpu/CMakeLists.txt
+++ b/offload/plugins-nextgen/amdgpu/CMakeLists.txt
@@ -4,7 +4,7 @@ find_package(hsa-runtime64 QUIET 1.2.0 HINTS ${CMAKE_INSTALL_PREFIX} PATHS /opt/
# Create the library and add the default arguments.
add_target_library(omptarget.rtl.amdgpu AMDGPU)
-target_sources(omptarget.rtl.amdgpu PRIVATE src/rtl.cpp)
+target_sources(omptarget.rtl.amdgpu PRIVATE src/rtl.cpp src/TaskGraphSchedule.cpp)
target_include_directories(omptarget.rtl.amdgpu PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/utils)
diff --git a/offload/plugins-nextgen/amdgpu/src/TaskGraphSchedule.cpp b/offload/plugins-nextgen/amdgpu/src/TaskGraphSchedule.cpp
new file mode 100644
index 00000000000000..0c4dc9a2757eaf
--- /dev/null
+++ b/offload/plugins-nextgen/amdgpu/src/TaskGraphSchedule.cpp
@@ -0,0 +1,353 @@
+//===-- TaskGraphSchedule.cpp - Lower a taskgraph onto in-order lanes -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// A single recursive pass over the region tree that places each leaf on a lane
+// and works out, for each one, the fewest completions it must wait for.
+//
+// What makes the waits few is that lanes are in-order, so knowing that one item
+// has completed is knowing that everything before it on its lane has too. Each
+// placed item therefore carries a *clock*: per lane, how many of that lane's
+// items are known to have completed by the time the item itself has. An item's
+// clock is the join of what it waits for and of the item before it on its own
+// lane, so knowledge flows along every ordering the lanes enforce. A producer
+// then needs no wait when what the consumer already knows covers it, which
+// subsumes three reductions:
+// * a dependence on something earlier on the *same* lane needs no wait;
+// * of several producers on one lane, waiting for the latest covers the rest;
+// * a producer that another producer, or the lane's earlier items, already
+// waited for needs no wait of its own.
+//
+// Correctness rests on three invariants, each of which is load-bearing (each
+// was a bug first, caught by the randomized oracle in
+// offload/unittests/TaskGraph):
+// 1. Positions are per-lane and only ever compared with positions, or clock
+// entries, for the same lane. An earlier producer does not imply a later
+// one has finished.
+// 2. Inside an irreducible region, predecessor signals are gathered
+// *unfiltered*. Reduction may only be applied where a leaf is actually
+// placed and its lane is therefore known: reducing against a guessed lane
+// drops real dependences for a successor sub-region whose several entry
+// leaves get spread over different lanes.
+// 3. Every per-lane structure is sized to the real lane count.
+//
+// Deadlock freedom follows from the single pass: an item can only be made to
+// wait for an already-placed producer, so the happens-before graph can only
+// ever point backwards in placement order and cannot contain a cycle.
+//
+//===----------------------------------------------------------------------===//
+
+#include "TaskGraphSchedule.h"
+
+#include "PluginInterface.h"
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Casting.h"
+
+#include <utility>
+#include <vector>
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+namespace {
+
+/// "This item has completed", carried from the item that produces it to the
+/// items that consume it, by item number.
+using SignalsTy = SmallVector<uint32_t, 8>;
+
+/// Per lane, how many of the lane's items are known to have completed.
+using ClockTy = SmallVector<uint32_t, 8>;
+
+/// An intra-region dependence edge, by child ordinal.
+using EdgeTy = std::pair<uint32_t, uint32_t>;
+
+/// Topologically sort [0, NumChildren) under \p Edges, predecessors first
+/// (Kahn's algorithm). A cycle is a hard error: the region tree is required to
+/// be a DAG, and silently picking some order for a cycle would produce a
+/// lowering that does not implement the graph.
+Error topoSort(uint32_t NumChildren, ArrayRef<EdgeTy> Edges,
+ SmallVectorImpl<uint32_t> &Order) {
+ SmallVector<uint32_t, 16> InDegree(NumChildren, 0);
+ for (const EdgeTy &E : Edges)
+ ++InDegree[E.second];
+
+ SmallVector<uint32_t, 16> Ready;
+ for (uint32_t I = 0; I < NumChildren; ++I)
+ if (InDegree[I] == 0)
+ Ready.push_back(I);
+
+ // Ready is consumed as a queue, not a stack, so the order comes out level by
+ // level. Any topological order is correct, but a breadth-first one emits a
+ // layer of mutually independent children consecutively, and the round-robin
+ // placement below then spreads that layer across the lanes. Depth-first
+ // instead emits a child right after its own predecessor, so the round robin
+ // puts it on the next lane where it can only wait: the lanes run in lockstep
+ // and the lowering extracts no concurrency at all.
+ for (uint32_t Head = 0; Head < Ready.size(); ++Head) {
+ uint32_t Idx = Ready[Head];
+ Order.push_back(Idx);
+ for (const EdgeTy &E : Edges)
+ if (E.first == Idx && --InDegree[E.second] == 0)
+ Ready.push_back(E.second);
+ }
+
+ if (Order.size() != NumChildren)
+ return Plugin::error(error::ErrorCode::INVALID_ARGUMENT,
+ "cyclic irreducible region in taskgraph");
+ return Plugin::success();
+}
+
+class LoweringTy {
+public:
+ LoweringTy(uint32_t NumLanes, TaskGraphLaneSinkTy &Sink)
+ : NumLanes(NumLanes), Sink(Sink), Tails(NumLanes, NoItem),
+ Lengths(NumLanes, 0) {}
+
+ /// Lower \p Elem, which must not start until every signal in \p In has fired.
+ /// \p Lane is the lane cursor, advanced in place; \p Out receives the
+ /// element's output signals, i.e. what a successor of it must wait for.
+ Error lower(const TaskGraphElement &Elem, ArrayRef<uint32_t> In,
+ uint32_t &Lane, SignalsTy &Out);
+
+private:
+ static constexpr uint32_t NoItem = ~0U;
+
+ /// What the lowering remembers of an item once it is placed.
+ struct PlacedTy {
+ uint32_t Lane;
+ /// 0-based position within Lane.
+ uint32_t Pos;
+ ClockTy Clock;
+ };
+
+ Error lowerIrreducible(const TaskGraphIrreducibleRegion &Region,
+ ArrayRef<uint32_t> In, uint32_t &Lane, SignalsTy &Out);
+
+ /// Place \p Node on \p Lane, waiting for whatever \p In reduces to.
+ /// Returns its item number.
+ Expected<uint32_t> place(const TaskGraphNode &Node, uint32_t Lane,
+ ArrayRef<uint32_t> In);
+
+ static void join(ClockTy &Into, const ClockTy &From) {
+ for (auto [I, F] : llvm::zip_equal(Into, From))
+ I = std::max(I, F);
+ }
+
+ const uint32_t NumLanes;
+ TaskGraphLaneSinkTy &Sink;
+ std::vector<PlacedTy> Placed;
+ /// Per lane, the last item placed on it, or NoItem. Sized to the real lane
+ /// count (invariant 3), as is every clock.
+ SmallVector<uint32_t, 8> Tails;
+ /// Per lane, how many items have been placed on it.
+ SmallVector<uint32_t, 8> Lengths;
+};
+
+Expected<uint32_t> LoweringTy::place(const TaskGraphNode &Node, uint32_t Lane,
+ ArrayRef<uint32_t> In) {
+ // What the item knows to have completed before it starts, without waiting
+ // for anything: on its own lane, everything placed there before it.
+ ClockTy Known(NumLanes, 0);
+ if (Tails[Lane] != NoItem)
+ Known = Placed[Tails[Lane]].Clock;
+
+ // Of the producers, the latest per lane not already known to have completed.
+ // Invariant 1: positions are only compared within a lane.
+ SmallVector<uint32_t, 8> Latest(NumLanes, NoItem);
+ for (uint32_t Sig : In) {
+ const PlacedTy &P = Placed[Sig];
+ if (Known[P.Lane] > P.Pos)
+ continue;
+ if (Latest[P.Lane] == NoItem || Placed[Latest[P.Lane]].Pos < P.Pos)
+ Latest[P.Lane] = Sig;
+ }
+
+ // Less any that another of them implies. Two cannot imply each other, which
+ // would make them each complete before the other, so dropping every implied
+ // one leaves a set that still implies them all.
+ SmallVector<uint32_t, 4> Waits;
+ for (uint32_t L = 0; L < NumLanes; ++L) {
+ if (Latest[L] == NoItem)
+ continue;
+ const uint32_t Pos = Placed[Latest[L]].Pos;
+ bool Implied = llvm::any_of(Latest, [&](uint32_t Other) {
+ return Other != NoItem && Other != Latest[L] &&
+ Placed[Other].Clock[L] > Pos;
+ });
+ if (!Implied)
+ Waits.push_back(Latest[L]);
+ }
+
+ // Everything in In completes before the item starts, waited for or implied,
+ // so its clock is all of theirs joined, plus the item itself.
+ ClockTy Clock = std::move(Known);
+ for (uint32_t Sig : In)
+ join(Clock, Placed[Sig].Clock);
+
+ const uint32_t Item = Placed.size();
+ const uint32_t Pos = Lengths[Lane]++;
+ Clock[Lane] = Pos + 1;
+ Tails[Lane] = Item;
+ Placed.push_back({Lane, Pos, std::move(Clock)});
+
+ if (Error Err = Sink.place(Item, Node, Lane, Waits))
+ return std::move(Err);
+ return Item;
+}
+
+Error LoweringTy::lower(const TaskGraphElement &Elem, ArrayRef<uint32_t> In,
+ uint32_t &Lane, SignalsTy &Out) {
+ // A leaf is the only thing that emits work. It stays on the current lane
+ // (the cursor is advanced by the enclosing region, not here) and its output
+ // signal is its own completion.
+ if (const auto *Node = dyn_cast<TaskGraphNode>(&Elem)) {
+ Expected<uint32_t> Item = place(*Node, Lane, In);
+ if (!Item)
+ return Item.takeError();
+ Out.push_back(*Item);
+ return Plugin::success();
+ }
+
+ const auto *Region = cast<TaskGraphRegion>(&Elem);
+ switch (Region->getKind()) {
+ case TGKind::ParRegion:
+ // Children are independent: each gets the region's own incoming
+ // dependences, and each starts on a fresh lane. With more children than
+ // lanes the surplus shares lanes and is serialized by in-order execution --
+ // a loss of concurrency, not of correctness.
+ for (const TaskGraphElement *Child : Region->children()) {
+ if (Error Err = lower(*Child, In, Lane, Out))
+ return Err;
+ Lane = (Lane + 1) % NumLanes;
+ }
+ break;
+
+ case TGKind::SeqRegion:
+ case TGKind::ExclRegion: {
+ // Thread the signals through the children left to right, deliberately
+ // leaving the lane cursor alone: a run of leaves then lands in order on one
+ // lane, where in-order execution orders them for free.
+ //
+ // An exclusive region is serialized exactly like a sequential one. A total
+ // order over the children trivially satisfies mutual exclusion; it gives up
+ // the freedom to run them in any order, which this backend has no way to
+ // exploit anyway.
+ SignalsTy Cur(In.begin(), In.end());
+ for (const TaskGraphElement *Child : Region->children()) {
+ SignalsTy Next;
+ if (Error Err = lower(*Child, Cur, Lane, Next))
+ return Err;
+ Cur = std::move(Next);
+ }
+ Out.append(Cur.begin(), Cur.end());
+ break;
+ }
+
+ case TGKind::IrreducibleRegion:
+ if (Error Err = lowerIrreducible(cast<TaskGraphIrreducibleRegion>(*Region),
+ In, Lane, Out))
+ return Err;
+ break;
+
+ default:
+ return Plugin::error(error::ErrorCode::INVALID_ARGUMENT,
+ "unexpected element kind in taskgraph region tree");
+ }
+
+ // A region that emitted no work at all is transparent to dependences: it
+ // passes its incoming signals straight through, so a successor still waits
+ // for whatever preceded it. Without this, an empty region would cut the
+ // dependence chain wherever the work either side of it landed on different
+ // lanes.
+ if (Out.empty())
+ Out.append(In.begin(), In.end());
+ return Plugin::success();
+}
+
+Error LoweringTy::lowerIrreducible(const TaskGraphIrreducibleRegion &Region,
+ ArrayRef<uint32_t> In, uint32_t &Lane,
+ SignalsTy &Out) {
+ const uint32_t NumChildren = Region.NumChildren;
+
+ // Both endpoints name a child of this region; the region's own entries and
+ // exits are derived below from which children have no incoming or no outgoing
+ // edge, so the stream does not carry them. An out-of-range ordinal is a
+ // malformed stream, and ignoring the edge would silently drop a dependence,
+ // so decline the graph and let it run on the host instead.
+ SmallVector<EdgeTy, 16> Edges;
+ for (const TaskGraphEdge &E : Region.edges()) {
+ if (E.Src < 0 || E.Dst < 0 || static_cast<uint32_t>(E.Src) >= NumChildren ||
+ static_cast<uint32_t>(E.Dst) >= NumChildren)
+ return Plugin::error(
+ error::ErrorCode::INVALID_ARGUMENT,
+ "irreducible region edge does not name a child of the region");
+ Edges.push_back(
+ {static_cast<uint32_t>(E.Src), static_cast<uint32_t>(E.Dst)});
+ }
+
+ SmallVector<uint32_t, 16> Order;
+ if (Error Err = topoSort(NumChildren, Edges, Order))
+ return Err;
+
+ // Visiting in topological order means a child's predecessors have already
+ // been lowered, so their output signals are available when it needs them.
+ std::vector<SignalsTy> Outgoing(NumChildren);
+ for (uint32_t Idx : Order) {
+ SignalsTy Intra;
+ for (const EdgeTy &E : Edges)
+ if (E.second == Idx)
+ Intra.append(Outgoing[E.first].begin(), Outgoing[E.first].end());
+
+ // An entry child (no predecessor in this region) takes the region's own
+ // incoming dependences; every other child reaches them transitively through
+ // the entries. Note Intra is passed on unfiltered -- see invariant 2.
+ ArrayRef<uint32_t> Use = Intra.empty() ? In : ArrayRef<uint32_t>(Intra);
+ if (Error Err = lower(*Region.children()[Idx], Use, Lane, Outgoing[Idx]))
+ return Err;
+ Lane = (Lane + 1) % NumLanes;
+ }
+
+ // The region has completed once its exits have; a DAG always has at least
+ // one.
+ for (uint32_t Idx = 0; Idx < NumChildren; ++Idx) {
+ bool HasSuccessor = false;
+ for (const EdgeTy &E : Edges)
+ if (E.first == Idx) {
+ HasSuccessor = true;
+ break;
+ }
+ if (!HasSuccessor)
+ Out.append(Outgoing[Idx].begin(), Outgoing[Idx].end());
+ }
+ return Plugin::success();
+}
+
+} // namespace
+
+Error lowerTaskGraphOntoLanes(const TaskGraphTy &Graph, uint32_t NumLanes,
+ TaskGraphLaneSinkTy &Sink) {
+ if (NumLanes == 0)
+ return Plugin::error(error::ErrorCode::INVALID_ARGUMENT,
+ "taskgraph lowering needs at least one lane");
+ if (!Graph.getRoot())
+ return Plugin::error(error::ErrorCode::INVALID_ARGUMENT,
+ "taskgraph has no root region");
+
+ LoweringTy Lowering(NumLanes, Sink);
+ SignalsTy Out;
+ uint32_t Lane = 0;
+ return Lowering.lower(*Graph.getRoot(), {}, Lane, Out);
+}
+
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
diff --git a/offload/plugins-nextgen/amdgpu/src/TaskGraphSchedule.h b/offload/plugins-nextgen/amdgpu/src/TaskGraphSchedule.h
new file mode 100644
index 00000000000000..926a6d50e67fb1
--- /dev/null
+++ b/offload/plugins-nextgen/amdgpu/src/TaskGraphSchedule.h
@@ -0,0 +1,83 @@
+//===-- TaskGraphSchedule.h - Lower a taskgraph onto in-order lanes -*- C++
+//-*-//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Lowering of the shared processed-taskgraph region tree (Shared/TaskGraph.h)
+// onto a fixed set of in-order lanes, each of which the plugin backs with a
+// stream over one HSA queue.
+//
+// This is a plugin-local policy, not part of the shared representation: it is
+// the right lowering for a backend whose only ordering primitives are in-order
+// queues plus completion signals, and the wrong one for a backend with a native
+// graph API, which would translate the same tree into a native DAG and never
+// assign lanes at all.
+//
+// The lowering decides where each leaf goes and what it must wait for, and
+// hands each leaf to a sink as soon as that is decided. What a placed leaf
+// becomes -- packets on a stream, a record a test checks -- is the sink's
+// business, so the algorithm is shared without a schedule in between.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PLUGINS_NEXTGEN_AMDGPU_TASKGRAPHSCHEDULE_H
+#define OFFLOAD_PLUGINS_NEXTGEN_AMDGPU_TASKGRAPHSCHEDULE_H
+
+#include "Shared/TaskGraph.h"
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/Error.h"
+
+#include <cstdint>
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+/// What a lowering onto lanes delivers its output to.
+///
+/// Every leaf of the graph is placed exactly once, as an *item*. Items are
+/// numbered from 0 in the order they are placed, and that is the number by
+/// which a later item's waits name them. Placement order is a topological
+/// order: an item only ever waits for items placed before it.
+class TaskGraphLaneSinkTy {
+public:
+ virtual ~TaskGraphLaneSinkTy() = default;
+
+ /// Place \p Leaf as item \p Item, at the end of \p Lane. It must not start
+ /// before every item in \p Waits has completed.
+ ///
+ /// \p Waits is minimal for in-order lanes. It never names an item on
+ /// \p Lane, since lane order covers those; it names at most one item per
+ /// other lane, the latest one needed; and it names no item that another of
+ /// its waits, or the lane's own earlier items, already imply has completed.
+ virtual Error place(uint32_t Item, const TaskGraphNode &Leaf, uint32_t Lane,
+ ArrayRef<uint32_t> Waits) = 0;
+};
+
+/// Lower \p Graph's region tree onto \p NumLanes in-order lanes, placing its
+/// leaves into \p Sink.
+///
+/// Running every lane concurrently, in order within each lane, with each item
+/// started only once its waits have completed, respects every dependence the
+/// region tree implies.
+///
+/// Fails on a malformed graph (no root, a cyclic irreducible region, zero
+/// lanes), and with whatever error the sink returns, at which point lowering
+/// stops. Any well-formed tree can be lowered onto any positive number of
+/// lanes: with fewer lanes than available concurrency the excess is serialized
+/// by sharing lanes, which costs parallelism but stays correct.
+Error lowerTaskGraphOntoLanes(const TaskGraphTy &Graph, uint32_t NumLanes,
+ TaskGraphLaneSinkTy &Sink);
+
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OFFLOAD_PLUGINS_NEXTGEN_AMDGPU_TASKGRAPHSCHEDULE_H
diff --git a/offload/unittests/CMakeLists.txt b/offload/unittests/CMakeLists.txt
index 151aef8338e3b0..e97f8e6643f12e 100644
--- a/offload/unittests/CMakeLists.txt
+++ b/offload/unittests/CMakeLists.txt
@@ -179,3 +179,8 @@ set(OFFLOAD_TESTS_FORCE_AMDGPU_ARCH "" CACHE STRING
add_subdirectory(OffloadAPI)
add_subdirectory(Conformance)
+
+# Host-only, and the code under test lives in the AMDGPU plugin.
+if("amdgpu" IN_LIST LIBOMPTARGET_PLUGINS_TO_BUILD)
+ add_subdirectory(TaskGraph)
+endif()
diff --git a/offload/unittests/TaskGraph/CMakeLists.txt b/offload/unittests/TaskGraph/CMakeLists.txt
new file mode 100644
index 00000000000000..06aa8ff899b9b9
--- /dev/null
+++ b/offload/unittests/TaskGraph/CMakeLists.txt
@@ -0,0 +1,23 @@
+# The lane lowering is pure graph code, so it is compiled straight into the test
+# rather than linked from the plugin archive: no device, no HSA, no libomptarget.
+# OffloadError.cpp comes along only for the error class the scheduler reports
+# through.
+set(LLVM_LINK_COMPONENTS Support)
+
+add_unittest(OffloadUnitTests "TaskGraph.unittests"
+ TaskGraphScheduleTest.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../plugins-nextgen/amdgpu/src/TaskGraphSchedule.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../plugins-nextgen/common/src/OffloadError.cpp
+)
+
+add_dependencies(TaskGraph.unittests PluginErrcodes)
+
+target_include_directories(TaskGraph.unittests PRIVATE
+ ${CMAKE_CURRENT_SOURCE_DIR}/../../plugins-nextgen/amdgpu/src
+ $<TARGET_PROPERTY:PluginCommon,INTERFACE_INCLUDE_DIRECTORIES>
+)
+
+target_compile_definitions(TaskGraph.unittests PRIVATE
+ TARGET_NAME=TaskGraphSchedule
+ DEBUG_PREFIX="TaskGraphSchedule"
+)
diff --git a/offload/unittests/TaskGraph/TaskGraphScheduleTest.cpp b/offload/unittests/TaskGraph/TaskGraphScheduleTest.cpp
new file mode 100644
index 00000000000000..28b6cacc0dfc5d
--- /dev/null
+++ b/offload/unittests/TaskGraph/TaskGraphScheduleTest.cpp
@@ -0,0 +1,829 @@
+//===-- TaskGraphScheduleTest.cpp - Tests for taskgraph lane lowering -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// What has to be true of lowerTaskGraphOntoLanes's output is that the
+// dependence order the region tree asks for is really enforced by the lanes it
+// places leaves on. Reading the placements and convincing yourself of that by
+// eye does not scale, so each test here does it mechanically:
+//
+// 1. Build a region tree and lower it onto N lanes, into a sink that records
+// every placement.
+// 2. Reconstruct the happens-before relation the placements actually
+// implement: consecutive items on a lane (in-order execution), plus each
+// item's waits.
+// 3. Derive the *intended* partial order over leaves straight from the tree,
+// with an oracle that shares no code with the lowering.
+// 4. Assert every intended dependence is reachable in the reconstructed
+// relation, that the relation is acyclic (i.e. cannot deadlock), and that
+// no wait is one the lanes already imply.
+//
+// The interesting cases are the ones where the reductions the lowering relies
+// on could drop a real dependence: irreducible regions, irreducible regions
+// nested inside each other, and multi-entry / multi-exit sub-regions as edge
+// endpoints. Every such case is run at several lane counts, including one lane
+// and more lanes than the tree can use, because the reductions behave
+// differently as items collide on lanes. A randomized sweep over generated
+// trees covers the shapes nobody thought to write down.
+//
+//===----------------------------------------------------------------------===//
+
+#include "TaskGraphSchedule.h"
+
+#include "Shared/TaskGraphBuilder.h"
+
+#include "llvm/ADT/DenseMap.h"
+
+#include "gtest/gtest.h"
+
+#include <cstdint>
+#include <map>
+#include <random>
+#include <set>
+#include <string>
+#include <utility>
+#include <vector>
+
+using namespace llvm;
+using namespace llvm::omp::target;
+using namespace llvm::omp::target::plugin;
+
+namespace {
+
+//===----------------------------------------------------------------------===//
+// Writing region trees
+//===----------------------------------------------------------------------===//
+
+/// A region tree as a test writes it. Leaves carry a small integer identity so
+/// the assertions can talk about "leaf 3" rather than about pointers.
+struct SpecTy {
+ enum KindTy {
+ Target,
+ EnterData,
+ ExitData,
+ UpdateData,
+ Par,
+ Seq,
+ Excl,
+ Irred
+ };
+
+ KindTy Kind;
+ int Id = 0; // leaves only
+ std::vector<SpecTy> Children; // regions only
+ std::vector<std::pair<int, int>> Edges; // Irred only, by child ordinal
+
+ bool isLeaf() const { return Kind <= UpdateData; }
+};
+
+SpecTy target(int Id) { return {SpecTy::Target, Id, {}, {}}; }
+SpecTy enterData(int Id) { return {SpecTy::EnterData, Id, {}, {}}; }
+SpecTy exitData(int Id) { return {SpecTy::ExitData, Id, {}, {}}; }
+SpecTy updateData(int Id) { return {SpecTy::UpdateData, Id, {}, {}}; }
+SpecTy par(std::vector<SpecTy> Children) {
+ return {SpecTy::Par, 0, std::move(Children), {}};
+}
+SpecTy seq(std::vector<SpecTy> Children) {
+ return {SpecTy::Seq, 0, std::move(Children), {}};
+}
+SpecTy excl(std::vector<SpecTy> Children) {
+ return {SpecTy::Excl, 0, std::move(Children), {}};
+}
+SpecTy irred(std::vector<SpecTy> Children,
+ std::vector<std::pair<int, int>> Edges) {
+ return {SpecTy::Irred, 0, std::move(Children), std::move(Edges)};
+}
+
+/// A TaskGraphTy materialized from a SpecTy, streamed in exactly as libomp
+/// streams a recorded graph into libomptarget: a measuring pass that only sizes
+/// the block, then a build pass into a block of that size. Going through the
+/// real construction path means these tests would also catch a tree that is
+/// mis-sized or mis-linked, not just a mis-scheduled one.
+class BuiltGraphTy {
+public:
+ explicit BuiltGraphTy(const SpecTy &Spec) {
+ Graph = TaskGraphBuilder::start(/*DeviceId=*/0, /*GraphId=*/0,
+ /*ByteSize=*/0);
+ stream(Spec);
+ size_t Bytes = TaskGraphBuilder(*Graph).finish();
+ delete Graph;
+
+ Graph = TaskGraphBuilder::start(/*DeviceId=*/0, /*GraphId=*/0, Bytes);
+ stream(Spec);
+ TaskGraphBuilder(*Graph).finish();
+ }
+ ~BuiltGraphTy() { delete Graph; }
+
+ const TaskGraphTy &graph() const { return *Graph; }
+
+ /// Why the graph is not well formed, or null.
+ const char *whyMalformed() const {
+ return TaskGraphBuilder::whyMalformed(*Graph);
+ }
+
+ /// The spec-level identity of a leaf, or -1 if it is not one of ours.
+ int idOf(const TaskGraphNode *Node) const {
+ auto It = Ids.find(Node);
+ return It == Ids.end() ? -1 : It->second;
+ }
+
+private:
+ template <class NodeT> void streamLeaf(int Id) {
+ TaskGraphBuilder B(*Graph);
+ if (NodeT *Node = B.makeLeaf<NodeT>()) {
+ B.linkChild(Node);
+ Ids[Node] = Id;
+ }
+ }
+
+ template <class RegionT> void streamRegion(const SpecTy &Spec) {
+ TaskGraphBuilder B(*Graph);
+ auto *Region =
+ B.makeRegion<RegionT>(static_cast<int32_t>(Spec.Children.size()));
+ if (Region)
+ B.pushRegion(Region);
+ for (const SpecTy &Child : Spec.Children)
+ stream(Child);
+ if (Region)
+ B.popRegion();
+ }
+
+ void stream(const SpecTy &Spec) {
+ switch (Spec.Kind) {
+ case SpecTy::Target:
+ return streamLeaf<TaskGraphTargetNode>(Spec.Id);
+ case SpecTy::EnterData:
+ return streamLeaf<TaskGraphTargetEnterDataNode>(Spec.Id);
+ case SpecTy::ExitData:
+ return streamLeaf<TaskGraphTargetExitDataNode>(Spec.Id);
+ case SpecTy::UpdateData:
+ return streamLeaf<TaskGraphTargetUpdateDataNode>(Spec.Id);
+ case SpecTy::Par:
+ return streamRegion<TaskGraphParRegion>(Spec);
+ case SpecTy::Seq:
+ return streamRegion<TaskGraphSeqRegion>(Spec);
+ case SpecTy::Excl:
+ return streamRegion<TaskGraphExclRegion>(Spec);
+ case SpecTy::Irred:
+ break;
+ }
+
+ // An irreducible region's edge array is sized with the region, so the
+ // measuring pass accounts for it before any edge is emitted.
+ TaskGraphBuilder B(*Graph);
+ auto *Region =
+ B.makeIrreducibleRegion(static_cast<int32_t>(Spec.Children.size()),
+ static_cast<int32_t>(Spec.Edges.size()));
+ if (Region)
+ B.pushRegion(Region);
+ for (const SpecTy &Child : Spec.Children)
+ stream(Child);
+ if (Region) {
+ for (const std::pair<int, int> &Edge : Spec.Edges)
+ B.addEdge(Edge.first, Edge.second);
+ B.popRegion();
+ }
+ }
+
+ TaskGraphTy *Graph = nullptr;
+ DenseMap<const TaskGraphNode *, int> Ids;
+};
+
+//===----------------------------------------------------------------------===//
+// What the lowering actually enforces
+//===----------------------------------------------------------------------===//
+
+/// Every placement a lowering made, as a sink saw them.
+struct PlacementsTy {
+ struct ItemTy {
+ const TaskGraphNode *Node = nullptr;
+ uint32_t Lane = 0;
+ std::vector<uint32_t> Waits;
+ };
+ std::vector<ItemTy> Items;
+ /// Per lane, the items placed on it in order.
+ std::vector<std::vector<uint32_t>> Lanes;
+};
+
+/// A sink that only records what it is given, and checks the numbering it is
+/// promised.
+class RecordingSinkTy final : public TaskGraphLaneSinkTy {
+public:
+ explicit RecordingSinkTy(uint32_t NumLanes) { P.Lanes.resize(NumLanes); }
+
+ Error place(uint32_t Item, const TaskGraphNode &Leaf, uint32_t Lane,
+ ArrayRef<uint32_t> Waits) override {
+ EXPECT_EQ(Item, P.Items.size()) << "items are numbered as they are placed";
+ P.Items.push_back(
+ {&Leaf, Lane, std::vector<uint32_t>(Waits.begin(), Waits.end())});
+ P.Lanes[Lane].push_back(Item);
+ return Error::success();
+ }
+
+ PlacementsTy P;
+};
+
+/// Lower \p Built onto \p NumLanes lanes and return what was placed.
+Expected<PlacementsTy> lowerGraph(const TaskGraphTy &Graph, uint32_t NumLanes) {
+ RecordingSinkTy Sink(NumLanes);
+ if (Error Err = lowerTaskGraphOntoLanes(Graph, NumLanes, Sink))
+ return std::move(Err);
+ return std::move(Sink.P);
+}
+
+/// The happens-before relation implied by a lowering, over its items.
+struct HappensBeforeTy {
+ std::vector<std::vector<uint32_t>> Succs;
+ /// Per item, its predecessor on its own lane, or ~0U.
+ std::vector<uint32_t> LanePred;
+ /// Leaf identity -> item, for talking about the lowering in spec terms.
+ std::map<int, uint32_t> ItemOfLeaf;
+
+ size_t size() const { return Succs.size(); }
+};
+
+HappensBeforeTy buildHappensBefore(const PlacementsTy &P,
+ const BuiltGraphTy &Built) {
+ HappensBeforeTy HB;
+ HB.Succs.resize(P.Items.size());
+ HB.LanePred.assign(P.Items.size(), ~0U);
+
+ // In-order execution within a lane.
+ for (const std::vector<uint32_t> &Lane : P.Lanes)
+ for (size_t I = 1; I < Lane.size(); ++I) {
+ HB.Succs[Lane[I - 1]].push_back(Lane[I]);
+ HB.LanePred[Lane[I]] = Lane[I - 1];
+ }
+
+ // Explicit waits, cross-lane or on an item placed on no lane.
+ for (uint32_t I = 0; I < P.Items.size(); ++I)
+ for (uint32_t Wait : P.Items[I].Waits)
+ HB.Succs[Wait].push_back(I);
+
+ for (uint32_t I = 0; I < P.Items.size(); ++I)
+ HB.ItemOfLeaf[Built.idOf(P.Items[I].Node)] = I;
+ return HB;
+}
+
+bool reachable(const HappensBeforeTy &HB, uint32_t From, uint32_t To) {
+ if (From == To)
+ return true;
+ std::vector<bool> Seen(HB.size(), false);
+ std::vector<uint32_t> Stack{From};
+ while (!Stack.empty()) {
+ uint32_t Node = Stack.back();
+ Stack.pop_back();
+ if (Seen[Node])
+ continue;
+ Seen[Node] = true;
+ for (uint32_t Succ : HB.Succs[Node]) {
+ if (Succ == To)
+ return true;
+ Stack.push_back(Succ);
+ }
+ }
+ return false;
+}
+
+/// Does the schedule guarantee leaf \p A completes before leaf \p B starts?
+bool happensBefore(const HappensBeforeTy &HB, int A, int B) {
+ auto ItA = HB.ItemOfLeaf.find(A), ItB = HB.ItemOfLeaf.find(B);
+ if (ItA == HB.ItemOfLeaf.end() || ItB == HB.ItemOfLeaf.end())
+ return false;
+ return reachable(HB, ItA->second, ItB->second);
+}
+
+/// A cycle here would be a schedule that cannot make progress.
+bool isAcyclic(const HappensBeforeTy &HB) {
+ enum { White, Grey, Black };
+ std::vector<uint8_t> Colour(HB.size(), White);
+ // Iterative DFS over (node, next successor index) so a deep chain cannot
+ // overflow the stack.
+ for (uint32_t Root = 0; Root < HB.size(); ++Root) {
+ if (Colour[Root] != White)
+ continue;
+ std::vector<std::pair<uint32_t, size_t>> Stack{{Root, 0}};
+ Colour[Root] = Grey;
+ while (!Stack.empty()) {
+ auto &[Node, Next] = Stack.back();
+ if (Next == HB.Succs[Node].size()) {
+ Colour[Node] = Black;
+ Stack.pop_back();
+ continue;
+ }
+ uint32_t Succ = HB.Succs[Node][Next++];
+ if (Colour[Succ] == Grey)
+ return false;
+ if (Colour[Succ] == White) {
+ Colour[Succ] = Grey;
+ Stack.push_back({Succ, 0});
+ }
+ }
+ }
+ return true;
+}
+
+//===----------------------------------------------------------------------===//
+// What the tree asks for (independent oracle)
+//===----------------------------------------------------------------------===//
+
+/// The intended dependences of a sub-tree, plus the leaves at its boundary:
+/// Entry leaves are those nothing inside precedes (so an incoming dependence
+/// must precede all of them), Exit leaves those nothing inside follows.
+///
+/// This reads the region model's *meaning* directly and shares no code with the
+/// scheduler, which is what makes it worth checking against.
+struct AnalysisTy {
+ std::vector<int> Entry;
+ std::vector<int> Exit;
+ std::set<std::pair<int, int>> Edges;
+};
+
+AnalysisTy analyze(const SpecTy &Spec) {
+ AnalysisTy Result;
+ if (Spec.isLeaf()) {
+ Result.Entry = {Spec.Id};
+ Result.Exit = {Spec.Id};
+ return Result;
+ }
+
+ std::vector<AnalysisTy> Children;
+ for (const SpecTy &Child : Spec.Children)
+ Children.push_back(analyze(Child));
+ for (const AnalysisTy &Child : Children)
+ Result.Edges.insert(Child.Edges.begin(), Child.Edges.end());
+
+ auto cross = [&](const std::vector<int> &From, const std::vector<int> &To) {
+ for (int A : From)
+ for (int B : To)
+ Result.Edges.insert({A, B});
+ };
+
+ switch (Spec.Kind) {
+ case SpecTy::Par:
+ // Independent children: the region's boundary is the union of theirs.
+ for (const AnalysisTy &Child : Children) {
+ Result.Entry.insert(Result.Entry.end(), Child.Entry.begin(),
+ Child.Entry.end());
+ Result.Exit.insert(Result.Exit.end(), Child.Exit.begin(),
+ Child.Exit.end());
+ }
+ break;
+
+ case SpecTy::Seq:
+ case SpecTy::Excl: {
+ // A chain: each child's exits precede the next child's entries. An
+ // exclusive region is required to serialize too, so it is analyzed the same
+ // way -- a total order over the children satisfies mutual exclusion.
+ //
+ // A child with no leaves at all is skipped rather than treated as a break
+ // in the chain, so the dependence either side of it is still required.
+ std::vector<int> PrevExits;
+ for (const AnalysisTy &Child : Children) {
+ if (Child.Entry.empty())
+ continue;
+ cross(PrevExits, Child.Entry);
+ if (Result.Entry.empty())
+ Result.Entry = Child.Entry;
+ PrevExits = Child.Exit;
+ }
+ Result.Exit = PrevExits;
+ break;
+ }
+
+ default: {
+ // Irreducible: the explicit edges are between whole children, so they
+ // become every (exit of source, entry of destination) pair. A child with
+ // no incoming edge is on the region's entry boundary, one with no outgoing
+ // edge on its exit boundary.
+ const size_t N = Children.size();
+ std::vector<bool> HasPred(N, false), HasSucc(N, false);
+ for (const std::pair<int, int> &Edge : Spec.Edges) {
+ cross(Children[Edge.first].Exit, Children[Edge.second].Entry);
+ HasSucc[Edge.first] = true;
+ HasPred[Edge.second] = true;
+ }
+ for (size_t I = 0; I < N; ++I) {
+ if (!HasPred[I])
+ Result.Entry.insert(Result.Entry.end(), Children[I].Entry.begin(),
+ Children[I].Entry.end());
+ if (!HasSucc[I])
+ Result.Exit.insert(Result.Exit.end(), Children[I].Exit.begin(),
+ Children[I].Exit.end());
+ }
+ break;
+ }
+ }
+ return Result;
+}
+
+void collectLeaves(const SpecTy &Spec, std::vector<int> &Out) {
+ if (Spec.isLeaf()) {
+ Out.push_back(Spec.Id);
+ return;
+ }
+ for (const SpecTy &Child : Spec.Children)
+ collectLeaves(Child, Out);
+}
+
+//===----------------------------------------------------------------------===//
+// The assertion every test is built from
+//===----------------------------------------------------------------------===//
+
+/// Check the shape of every item's waits: they name only items placed before
+/// it, none on its own lane, at most one per other lane, and none that the
+/// item's lane or its other waits already imply. That last one is what makes
+/// the waits minimal, and is the property the lowering's clocks exist for.
+void checkWaits(const PlacementsTy &P, const HappensBeforeTy &HB,
+ const std::string &Context) {
+ for (uint32_t I = 0; I < P.Items.size(); ++I) {
+ const PlacementsTy::ItemTy &Item = P.Items[I];
+ std::set<uint32_t> LanesWaited;
+ for (uint32_t Wait : Item.Waits) {
+ EXPECT_LT(Wait, I) << Context << ": item " << I
+ << " waits for an item placed after it";
+ if (Wait >= I)
+ continue;
+ const uint32_t WaitLane = P.Items[Wait].Lane;
+ EXPECT_NE(WaitLane, Item.Lane)
+ << Context << ": item " << I << " waits on its own lane";
+ EXPECT_TRUE(LanesWaited.insert(WaitLane).second)
+ << Context << ": item " << I << " waits twice on lane " << WaitLane;
+
+ // Implied by the item before it on its lane, or by another wait.
+ std::vector<uint32_t> Others(Item.Waits.begin(), Item.Waits.end());
+ if (HB.LanePred[I] != ~0U)
+ Others.push_back(HB.LanePred[I]);
+ for (uint32_t Other : Others)
+ EXPECT_TRUE(Other == Wait || !reachable(HB, Wait, Other))
+ << Context << ": item " << I << " waits for " << Wait << ", which "
+ << Other << " already implies";
+ }
+ }
+}
+
+/// Lower \p Spec onto \p NumLanes lanes and check the result against the
+/// oracle. \p Context names the case in any failure message.
+HappensBeforeTy checkSchedule(const SpecTy &Spec, uint32_t NumLanes,
+ const std::string &Context) {
+ BuiltGraphTy Built(Spec);
+ EXPECT_EQ(Built.whyMalformed(), nullptr)
+ << Context << ": well-formed graph rejected";
+ auto POrErr = lowerGraph(Built.graph(), NumLanes);
+ EXPECT_TRUE(static_cast<bool>(POrErr)) << Context << ": lowering failed";
+ if (!POrErr) {
+ consumeError(POrErr.takeError());
+ return {};
+ }
+
+ HappensBeforeTy HB = buildHappensBefore(*POrErr, Built);
+
+ std::vector<int> Leaves;
+ collectLeaves(Spec, Leaves);
+ EXPECT_EQ(POrErr->Items.size(), Leaves.size())
+ << Context << ": every leaf must be placed exactly once";
+ for (int Leaf : Leaves)
+ EXPECT_TRUE(HB.ItemOfLeaf.count(Leaf))
+ << Context << ": leaf " << Leaf << " was not placed";
+
+ for (const std::pair<int, int> &Edge : analyze(Spec).Edges)
+ EXPECT_TRUE(happensBefore(HB, Edge.first, Edge.second))
+ << Context << ": dependence " << Edge.first << " -> " << Edge.second
+ << " is not enforced on " << NumLanes << " lanes";
+
+ EXPECT_TRUE(isAcyclic(HB)) << Context << ": lowering can deadlock";
+ checkWaits(*POrErr, HB, Context);
+ return HB;
+}
+
+/// Same, over a spread of lane counts: one lane (everything collides), a couple
+/// of lanes (items collide selectively), and more lanes than the tree can use.
+void checkScheduleAcrossLanes(const SpecTy &Spec, const std::string &Context,
+ std::vector<uint32_t> LaneCounts = {1, 2, 3, 4, 5,
+ 8}) {
+ for (uint32_t NumLanes : LaneCounts)
+ checkSchedule(Spec, NumLanes, Context + "/" + std::to_string(NumLanes));
+}
+
+//===----------------------------------------------------------------------===//
+// Tests
+//===----------------------------------------------------------------------===//
+
+TEST(TaskGraphSchedule, LeavesAreScheduledOnce) {
+ SpecTy Spec = seq({target(1), enterData(2), exitData(3), updateData(4)});
+ BuiltGraphTy Built(Spec);
+ auto POrErr = lowerGraph(Built.graph(), 4);
+ ASSERT_TRUE(static_cast<bool>(POrErr));
+ EXPECT_EQ(POrErr->Items.size(), 4u);
+
+ // Every leaf kind reaches the sink as the leaf it was recorded as.
+ std::map<int, TGKind> Kinds;
+ for (const PlacementsTy::ItemTy &Item : POrErr->Items)
+ Kinds[Built.idOf(Item.Node)] = Item.Node->getKind();
+ EXPECT_EQ(Kinds[1], TGKind::TargetNode);
+ EXPECT_EQ(Kinds[2], TGKind::TargetEnterDataNode);
+ EXPECT_EQ(Kinds[3], TGKind::TargetExitDataNode);
+ EXPECT_EQ(Kinds[4], TGKind::TargetUpdateDataNode);
+}
+
+TEST(TaskGraphSchedule, SingleLeafNeedsNoWaits) {
+ SpecTy Spec = target(1);
+ BuiltGraphTy Built(Spec);
+ auto POrErr = lowerGraph(Built.graph(), 4);
+ ASSERT_TRUE(static_cast<bool>(POrErr));
+ ASSERT_EQ(POrErr->Items.size(), 1u);
+ EXPECT_TRUE(POrErr->Items[0].Waits.empty());
+}
+
+TEST(TaskGraphSchedule, SequentialRegionIsTotallyOrdered) {
+ SpecTy Spec = seq({target(1), target(2), target(3), target(4)});
+ HappensBeforeTy HB = checkSchedule(Spec, 4, "seq");
+ EXPECT_TRUE(happensBefore(HB, 1, 4));
+}
+
+TEST(TaskGraphSchedule, ExclusiveRegionSerializes) {
+ SpecTy Spec = excl({target(1), target(2), target(3)});
+ HappensBeforeTy HB = checkSchedule(Spec, 4, "excl");
+ EXPECT_TRUE(happensBefore(HB, 1, 3));
+}
+
+TEST(TaskGraphSchedule, ParallelRegionIsUnordered) {
+ SpecTy Spec = par({target(1), target(2), target(3), target(4)});
+ EXPECT_TRUE(analyze(Spec).Edges.empty());
+ HappensBeforeTy HB = checkSchedule(Spec, 4, "par");
+ // Unordered means unordered: nothing is forced either way.
+ EXPECT_FALSE(happensBefore(HB, 1, 2));
+ EXPECT_FALSE(happensBefore(HB, 2, 1));
+}
+
+TEST(TaskGraphSchedule, EveryParallelChildPrecedesASuccessor) {
+ // The join has to wait for children spread over several lanes.
+ SpecTy Spec = seq({par({target(1), target(2), target(3)}), target(4)});
+ checkScheduleAcrossLanes(Spec, "par-then-seq");
+ HappensBeforeTy HB = checkSchedule(Spec, 4, "par-then-seq/4");
+ for (int Child : {1, 2, 3})
+ EXPECT_TRUE(happensBefore(HB, Child, 4));
+}
+
+TEST(TaskGraphSchedule, IrreducibleDiamond) {
+ SpecTy Spec = irred({target(1), target(2), target(3), target(4)},
+ {{0, 1}, {0, 2}, {1, 3}, {2, 3}});
+ checkScheduleAcrossLanes(Spec, "diamond");
+ HappensBeforeTy HB = checkSchedule(Spec, 4, "diamond/4");
+ EXPECT_TRUE(happensBefore(HB, 1, 4));
+ // The two middle leaves are concurrent and must not be ordered.
+ EXPECT_FALSE(happensBefore(HB, 2, 3));
+ EXPECT_FALSE(happensBefore(HB, 3, 2));
+}
+
+TEST(TaskGraphSchedule, IrreducibleMultiPredecessorJoin) {
+ // Three producers into one consumer, then a further consumer: with fewer
+ // lanes than producers some of them share a lane, which is what the
+ // latest-producer-per-lane reduction has to get right.
+ SpecTy Spec = irred({target(1), target(2), target(3), target(4), target(5)},
+ {{0, 3}, {1, 3}, {2, 3}, {3, 4}});
+ checkScheduleAcrossLanes(Spec, "join");
+ HappensBeforeTy HB = checkSchedule(Spec, 2, "join/2");
+ for (int Producer : {1, 2, 3})
+ EXPECT_TRUE(happensBefore(HB, Producer, 4));
+ EXPECT_TRUE(happensBefore(HB, 1, 5));
+}
+
+TEST(TaskGraphSchedule, IrreducibleWithinIrreducible) {
+ // The case the same-lane elision is most likely to get wrong: each child of
+ // the outer region is itself irreducible, so an outer edge's endpoints are
+ // whole sets of leaves that end up on different lanes.
+ SpecTy A = irred({target(10), target(11), target(12), target(13)},
+ {{0, 1}, {0, 2}, {1, 3}, {2, 3}});
+ SpecTy B = irred({target(20), target(21), target(22)}, {{0, 1}, {0, 2}});
+ SpecTy C = irred({target(30), target(31), target(32)}, {{0, 2}, {1, 2}});
+ SpecTy Spec = irred({A, B, C}, {{0, 1}, {1, 2}});
+
+ checkScheduleAcrossLanes(Spec, "nested");
+ HappensBeforeTy HB = checkSchedule(Spec, 3, "nested/3");
+ EXPECT_TRUE(happensBefore(HB, 13, 20));
+ for (int Exit : {21, 22})
+ for (int Entry : {30, 31})
+ EXPECT_TRUE(happensBefore(HB, Exit, Entry));
+ EXPECT_TRUE(happensBefore(HB, 10, 32));
+}
+
+TEST(TaskGraphSchedule, DeepHeterogeneousNesting) {
+ auto stage = [](int Base) {
+ return irred({enterData(Base), target(Base + 1),
+ irred({target(Base + 2), target(Base + 3)}, {{0, 1}}),
+ exitData(Base + 4)},
+ {{0, 1}, {1, 2}, {2, 3}});
+ };
+ SpecTy Spec = irred({stage(40), stage(50)}, {{0, 1}});
+
+ checkScheduleAcrossLanes(Spec, "deep");
+ HappensBeforeTy HB = checkSchedule(Spec, 4, "deep/4");
+ EXPECT_TRUE(happensBefore(HB, 44, 50));
+ EXPECT_TRUE(happensBefore(HB, 40, 44));
+ EXPECT_TRUE(happensBefore(HB, 40, 54));
+}
+
+TEST(TaskGraphSchedule, ChainLongerThanLaneCount) {
+ // A chain of ten leaves on two lanes: consecutive leaves land on alternating
+ // lanes, so nearly every dependence needs a real wait.
+ std::vector<SpecTy> Children;
+ std::vector<std::pair<int, int>> Edges;
+ for (int I = 0; I < 10; ++I) {
+ Children.push_back(target(I + 1));
+ if (I)
+ Edges.push_back({I - 1, I});
+ }
+ SpecTy Spec = irred(std::move(Children), std::move(Edges));
+
+ HappensBeforeTy HB = checkSchedule(Spec, 2, "chain/2");
+ for (int I = 1; I < 10; ++I)
+ EXPECT_TRUE(happensBefore(HB, I, I + 1));
+ EXPECT_TRUE(happensBefore(HB, 1, 10));
+}
+
+TEST(TaskGraphSchedule, ParallelRegionAsEdgeSource) {
+ // The source of an irreducible edge is a region whose exits span lanes, so a
+ // single wait cannot cover it.
+ SpecTy Spec =
+ irred({par({target(1), target(2), target(3)}), target(4)}, {{0, 1}});
+ checkScheduleAcrossLanes(Spec, "par-source");
+ HappensBeforeTy HB = checkSchedule(Spec, 4, "par-source/4");
+ for (int Producer : {1, 2, 3})
+ EXPECT_TRUE(happensBefore(HB, Producer, 4));
+}
+
+TEST(TaskGraphSchedule, IrreducibleInParallelInIrreducible) {
+ SpecTy Inner = irred({target(10), target(11)}, {{0, 1}});
+ SpecTy Spec = irred({par({Inner, target(12)}), target(13)}, {{0, 1}});
+ checkScheduleAcrossLanes(Spec, "mixed");
+ HappensBeforeTy HB = checkSchedule(Spec, 5, "mixed/5");
+ EXPECT_TRUE(happensBefore(HB, 10, 11));
+ EXPECT_TRUE(happensBefore(HB, 11, 13));
+ EXPECT_TRUE(happensBefore(HB, 12, 13));
+}
+
+TEST(TaskGraphSchedule, SequenceOfMultiExitIrreducibleRegions) {
+ SpecTy First = irred({target(20), target(21), target(22)}, {{0, 1}, {0, 2}});
+ SpecTy Second = irred({target(30), target(31), target(32)}, {{0, 2}, {1, 2}});
+ SpecTy Spec = seq({First, Second});
+ checkScheduleAcrossLanes(Spec, "seq-of-irreds");
+ HappensBeforeTy HB = checkSchedule(Spec, 3, "seq-of-irreds/3");
+ for (int Exit : {21, 22})
+ for (int Entry : {30, 31})
+ EXPECT_TRUE(happensBefore(HB, Exit, Entry));
+}
+
+TEST(TaskGraphSchedule, WideFanIn) {
+ // Six producers into one consumer at lane counts that force several producers
+ // per lane, exercising the per-lane latest-producer reduction.
+ const int NumProducers = 6;
+ std::vector<SpecTy> Children;
+ std::vector<std::pair<int, int>> Edges;
+ for (int I = 0; I <= NumProducers; ++I)
+ Children.push_back(target(I + 1));
+ for (int I = 0; I < NumProducers; ++I)
+ Edges.push_back({I, NumProducers});
+ SpecTy Spec = irred(std::move(Children), std::move(Edges));
+
+ for (uint32_t NumLanes : {2u, 3u, 4u, 5u}) {
+ HappensBeforeTy HB =
+ checkSchedule(Spec, NumLanes, "fan-in/" + std::to_string(NumLanes));
+ for (int Producer = 1; Producer <= NumProducers; ++Producer)
+ EXPECT_TRUE(happensBefore(HB, Producer, NumProducers + 1));
+ }
+}
+
+TEST(TaskGraphSchedule, EmptyRegionPassesDependencesThrough) {
+ // A region that emits nothing must not cut the chain around it. The leaves
+ // either side have to be on different lanes for this to test anything: on one
+ // lane in-order execution would order them whatever the schedule says, so the
+ // predecessors here are a parallel region, which spreads them over lanes.
+ SpecTy Spec = seq({par({target(1), target(2)}), par({}), target(3)});
+ HappensBeforeTy HB = checkSchedule(Spec, 4, "empty-region");
+ EXPECT_TRUE(happensBefore(HB, 1, 3));
+ EXPECT_TRUE(happensBefore(HB, 2, 3));
+}
+
+TEST(TaskGraphSchedule, WaitImpliedByAnotherWaitIsDropped) {
+ // A triangle over three lanes: 2 waits for 1, and 3 for both. Keeping the
+ // latest producer per lane would leave 3 waiting for 1 and 2, though 2 has
+ // already waited for 1; one wait is all 3 needs.
+ SpecTy Spec =
+ irred({target(1), target(2), target(3)}, {{0, 1}, {0, 2}, {1, 2}});
+ BuiltGraphTy Built(Spec);
+ auto POrErr = lowerGraph(Built.graph(), 3);
+ ASSERT_TRUE(static_cast<bool>(POrErr));
+ HappensBeforeTy HB = buildHappensBefore(*POrErr, Built);
+ const PlacementsTy::ItemTy &Third = POrErr->Items[HB.ItemOfLeaf[3]];
+ ASSERT_EQ(Third.Lane, 2u) << "the triangle should span all three lanes";
+ EXPECT_EQ(Third.Waits, std::vector<uint32_t>{HB.ItemOfLeaf[2]});
+ checkScheduleAcrossLanes(Spec, "triangle");
+}
+
+TEST(TaskGraphSchedule, CyclicIrreducibleRegionIsRejected) {
+ // Picking some order for a cycle would silently produce a schedule that does
+ // not implement the graph, so this has to be an error.
+ for (const SpecTy &Spec :
+ {irred({target(1), target(2)}, {{0, 1}, {1, 0}}),
+ irred({target(1), target(2), target(3)}, {{0, 1}, {1, 2}, {2, 0}})}) {
+ BuiltGraphTy Built(Spec);
+ auto POrErr = lowerGraph(Built.graph(), 4);
+ EXPECT_FALSE(static_cast<bool>(POrErr));
+ if (!POrErr)
+ consumeError(POrErr.takeError());
+ }
+}
+
+TEST(TaskGraphStructure, WellFormedGraphIsAccepted) {
+ BuiltGraphTy Built(seq(
+ {target(1), irred({target(2), target(3), target(4)}, {{0, 1}, {0, 2}})}));
+ EXPECT_EQ(Built.whyMalformed(), nullptr);
+}
+
+TEST(TaskGraphStructure, MalformedGraphIsRejected) {
+ // libomptarget checks these once so that no backend has to, which is what
+ // keeps a backend from silently dropping a dependence it cannot place.
+ for (const SpecTy &Spec :
+ {irred({target(1), target(2)}, {{0, 1}, {1, 0}}),
+ irred({target(1), target(2), target(3)}, {{0, 1}, {1, 2}, {2, 0}}),
+ irred({target(1), target(2)}, {{0, 2}}),
+ irred({target(1), target(2)}, {{-1, 1}})}) {
+ BuiltGraphTy Built(Spec);
+ EXPECT_NE(Built.whyMalformed(), nullptr);
+ }
+}
+
+TEST(TaskGraphSchedule, ZeroLanesIsRejected) {
+ SpecTy Spec = target(1);
+ BuiltGraphTy Built(Spec);
+ auto POrErr = lowerGraph(Built.graph(), 0);
+ EXPECT_FALSE(static_cast<bool>(POrErr));
+ if (!POrErr)
+ consumeError(POrErr.takeError());
+}
+
+//===----------------------------------------------------------------------===//
+// Randomized sweep
+//===----------------------------------------------------------------------===//
+
+/// Generate an arbitrary region tree. Irreducible edge sets only go from lower
+/// to higher child ordinals, which keeps them acyclic (cycles are covered by
+/// their own test) while still producing every DAG shape.
+SpecTy generateSpec(std::mt19937 &Rng, int Depth, int &NextId) {
+ const int LeafPercent = Depth <= 0 ? 100 : 35;
+ if (static_cast<int>(Rng() % 100) < LeafPercent) {
+ int Id = ++NextId;
+ switch (Rng() % 4) {
+ case 0:
+ return target(Id);
+ case 1:
+ return enterData(Id);
+ case 2:
+ return updateData(Id);
+ default:
+ return exitData(Id);
+ }
+ }
+
+ const size_t NumChildren = 2 + Rng() % 3;
+ std::vector<SpecTy> Children;
+ for (size_t I = 0; I < NumChildren; ++I)
+ Children.push_back(generateSpec(Rng, Depth - 1, NextId));
+
+ switch (Rng() % 4) {
+ case 0:
+ return seq(std::move(Children));
+ case 1:
+ return par(std::move(Children));
+ case 2:
+ return excl(std::move(Children));
+ default: {
+ std::vector<std::pair<int, int>> Edges;
+ for (size_t I = 0; I < NumChildren; ++I)
+ for (size_t J = I + 1; J < NumChildren; ++J)
+ if (Rng() % 100 < 45)
+ Edges.push_back({static_cast<int>(I), static_cast<int>(J)});
+ return irred(std::move(Children), std::move(Edges));
+ }
+ }
+}
+
+TEST(TaskGraphSchedule, RandomizedTreesAgainstOracle) {
+ const unsigned NumTrees = 400;
+ const unsigned BaseSeed = 0xC0FFEE;
+ for (unsigned I = 0; I < NumTrees; ++I) {
+ const unsigned Seed = BaseSeed + I;
+ std::mt19937 Rng(Seed);
+ int NextId = 0;
+ SpecTy Spec = generateSpec(Rng, /*Depth=*/4, NextId);
+ for (uint32_t NumLanes : {1u, 2u, 3u, 5u, 8u})
+ checkSchedule(Spec, NumLanes, "seed=" + std::to_string(Seed));
+ }
+}
+
+} // namespace
>From 450e6ef470f5c5b1a8ee6a1df7ee025116772b91 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 1 Oct 2026 07:00:40 -0500
Subject: [PATCH 28/29] [Offload][AMDGPU] Taskgraph support in AMDGCN offload
plugin
This patch adds an implementation of taskgraph execution to the AMDGPU
offload plugin. Graphs are scheduled onto a set of in-order streams
(the plugin's existing abstraction) by the lowering in the previous
patch, which places each leaf through a TaskGraphLaneSinkTy that this
patch implements. Each "lane" in the terms of that algorithm maps
onto a distinct stream (which in turn maps onto its own AQL queue).
The current approach to mapping graphs onto GPU hardware splits the
execution of the graph according to the regions of its inferred nested
structure: maximal regions which are formed entirely from target regions
are passed to libomptarget, whereas host-only or mixed regions are run
using libomp's executor, with target constructs invoked synchronously
with respect to the host thread which claims that part of the graph.
A consequence of that is that target work is never queued behind host
work: because host tasks can contain calls that can queue GPU work
themselves, it is *generally* unsafe to do otherwise, else the code is
likely to deadlock.
Data mappings are handled by the "data plan" support in previous patches,
so this patch only needs to deal with transferring of data (not allocation
or deallocation of target memory, and not maintenance of host-to-device
mappings). The patch implements the pinning/unpinning hooks forming part
of the libomptarget<->plugin API.
This is the first backend to claim graphs, so the tests that came
earlier in the series marked XFAIL for want of one -- the data plan's,
and those of the other runtime changes only a claimed graph exercises --
pass from here, and their XFAILs are removed. New tests cover what is
specific to this backend.
Assisted-By: Claude
---
offload/plugins-nextgen/amdgpu/src/rtl.cpp | 842 ++++++++++++++++--
.../taskgraph_claim_finalize_partial.c | 3 -
.../test/offloading/taskgraph_decline_info.c | 3 -
.../offloading/taskgraph_exit_data_borrowed.c | 3 -
.../offloading/taskgraph_firstprivate_inner.c | 3 -
.../taskgraph_firstprivate_restage.c | 3 -
.../taskgraph_firstprivate_scalar.c | 3 -
.../taskgraph_firstprivate_scalar_reparam.c | 3 -
.../taskgraph_host_task_subgraph_order.c | 3 -
.../offloading/taskgraph_lambda_capture.cpp | 3 -
offload/test/offloading/taskgraph_map_drift.c | 3 -
.../taskgraph_map_dynamic_bound_saved.c | 3 -
.../offloading/taskgraph_map_partial_borrow.c | 3 -
.../taskgraph_map_relocated_mixed.c | 3 -
.../taskgraph_map_relocated_stack.c | 3 -
.../offloading/taskgraph_plan_dump_attach.c | 3 -
.../offloading/taskgraph_plan_dump_borrowed.c | 3 -
.../taskgraph_plan_dump_enter_exit.c | 3 -
.../taskgraph_plan_dump_firstprivate.c | 3 -
.../taskgraph_plan_dump_irreducible.c | 3 -
.../offloading/taskgraph_present_absent.c | 3 -
.../offloading/taskgraph_target_data_body.c | 3 -
.../taskgraph_target_data_body_visible.c | 3 -
.../taskgraph_target_encounter_order.c | 60 ++
.../taskgraph_target_mutex_irreducible.c | 3 -
.../taskgraph_target_plugin_backend.c | 94 ++
.../taskgraph_unordered_copy_decline.c | 3 -
.../taskgraph_unordered_copy_replay.c | 3 -
.../offloading/taskgraph_use_device_ptr.c | 3 -
29 files changed, 937 insertions(+), 137 deletions(-)
create mode 100644 offload/test/offloading/taskgraph_target_encounter_order.c
create mode 100644 offload/test/offloading/taskgraph_target_plugin_backend.c
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 07b060a8f203d4..1bdb7c3607a775 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -18,6 +18,7 @@
#include <deque>
#include <functional>
#include <mutex>
+#include <set>
#include <string>
#include <system_error>
#include <unistd.h>
@@ -28,15 +29,18 @@
#include "Shared/Debug.h"
#include "Shared/Environment.h"
#include "Shared/RefCnt.h"
+#include "Shared/TaskGraph.h"
#include "Shared/Utils.h"
#include "Utils/ELF.h"
#include "GlobalHandler.h"
#include "OffloadAPI.h"
#include "PluginInterface.h"
+#include "TaskGraphSchedule.h"
#include "UtilitiesRTL.h"
#include "omptarget.h"
+#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
@@ -682,6 +686,18 @@ struct AMDGPUKernelTy : public GenericKernelTy {
/// Get the size of implicitargs based on the code object version.
uint32_t getImplicitArgsSize() const { return ImplicitArgsSize; }
+ /// Get the kernarg segment size (explicit + implicit args).
+ uint32_t getArgsSize() const { return ArgsSize; }
+
+ /// Write the kernarg segment, getArgsSize() bytes at \p Buf: the \p NumArgs
+ /// explicit arguments at the offsets the kernel's metadata gives, each read
+ /// through \p Args, then the implicit arguments for the given geometry.
+ /// Shared by launchImpl and the taskgraph backend, which writes a node's
+ /// segment once and reuses it.
+ Error writeArgs(void *Buf, uint32_t NumArgs, void *const *Args,
+ const uint32_t NumThreads[3], const uint32_t NumBlocks[3],
+ uint32_t DynBlockMemSize) const;
+
/// Indicates whether or not we need to set up our own private segment size.
bool usesDynamicStack() const { return DynamicStack; }
@@ -1867,6 +1883,17 @@ struct AMDGPUEventTy {
return RecordedStream->synchronizeOn(*this);
}
+ /// Whether the recorded operation has completed. Does not block.
+ Expected<bool> query() {
+ std::lock_guard<std::mutex> Lock(Mutex);
+
+ // Nothing recorded, or nothing was outstanding when it was.
+ if (!RecordedStream || RecordedSlot < 0)
+ return true;
+
+ return RecordedStream->isEventComplete(*this);
+ }
+
/// Return the elapsed time in milliseconds between this event and EndEvent.
Expected<float> getElapsedTime(AMDGPUEventTy &EndEvent);
@@ -2423,9 +2450,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
OMPX_StreamBusyWait("LIBOMPTARGET_AMDGPU_STREAM_BUSYWAIT", 2000000),
OMPX_UseMultipleSdmaEngines(
"LIBOMPTARGET_AMDGPU_USE_MULTIPLE_SDMA_ENGINES", false),
- OMPX_ApuMaps("OMPX_APU_MAPS", false), AMDGPUStreamManager(*this, Agent),
- AMDGPUEventManager(*this), AMDGPUSignalManager(*this), Agent(Agent),
- HostDevice(HostDevice) {}
+ OMPX_ApuMaps("OMPX_APU_MAPS", false),
+ OMPX_TaskGraph("LIBOMPTARGET_AMDGPU_TASKGRAPH", true),
+ OMPX_TaskGraphLanes("LIBOMPTARGET_AMDGPU_TASKGRAPH_LANES", 0),
+ AMDGPUStreamManager(*this, Agent), AMDGPUEventManager(*this),
+ AMDGPUSignalManager(*this), Agent(Agent), HostDevice(HostDevice) {}
~AMDGPUDeviceTy() {}
@@ -3717,6 +3746,38 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
bool useMultipleSdmaEngines() const { return OMPX_UseMultipleSdmaEngines; }
+ //===----- taskgraph replay backend --------------------------------===//
+ // The taskgraph path, taken for every graph this backend can lower.
+ // LIBOMPTARGET_AMDGPU_TASKGRAPH=0 turns it off and leaves libomp to issue
+ // the graph's target constructs itself, which is how a problem in this
+ // backend is told apart from one above it.
+ //
+ // libomptarget's data plan decides where each leaf's data lives and
+ // provides any device storage the graph needs, so this backend only moves
+ // data. Finalize does the rest of the interpretation once: the shared
+ // region tree is lowered onto a pool of stream "lanes"
+ // (lowerTaskGraphOntoLanes), each on its own AQL queue; the plan's staging
+ // bytes are locked for the graph's lifetime; and each kernel launch gets a
+ // ready-to-copy kernarg image. A replay locks the host ranges its copies
+ // touch (pinTaskGraph), adopts whatever the plan reports changed
+ // (reparametrizeTaskGraph), then re-issues the fixed schedule through the
+ // plugin's ordinary stream primitives, gating cross-lane edges with
+ // AMDGPUEventTy, and drains the lanes before returning.
+ //
+ // Finalize declines (returns an error, which is not a failure) when the
+ // graph contains something this lowering cannot express; libomp then
+ // issues the graph's target constructs itself.
+ bool supportsTaskGraphs() const override { return OMPX_TaskGraph; }
+ Expected<void *> finalizeTaskGraph(const TaskGraphTy *Graph) override;
+ Error replayTaskGraph(const TaskGraphTy *Graph) override;
+ Error destroyTaskGraph(const TaskGraphTy *Graph) override;
+ Error reparametrizeTaskGraph(
+ const TaskGraphTy *Graph,
+ ArrayRef<llvm::omp::target::TaskGraphArgUpdateTy> Updates,
+ bool Resync) override;
+ Error pinTaskGraph(const TaskGraphTy *Graph) override;
+ Error unpinTaskGraph(const TaskGraphTy *Graph) override;
+
private:
using AMDGPUEventRef = AMDGPUResourceRef<AMDGPUEventTy>;
using AMDGPUEventManagerTy = GenericDeviceResourceManagerTy<AMDGPUEventRef>;
@@ -3841,6 +3902,17 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
/// automatic zero-copy behavior on non-APU GPUs.
BoolEnvar OMPX_ApuMaps;
+ /// Switch for the taskgraph replay backend, read by supportsTaskGraphs().
+ /// On by default; set to 0 to have this plugin claim no graphs, in which
+ /// case libomp issues each graph's target constructs itself.
+ BoolEnvar OMPX_TaskGraph;
+
+ /// Number of stream "lanes" the taskgraph backend distributes nodes over. 0
+ /// (the default) means "use the HSA queue count" so each lane lands on a
+ /// distinct AQL queue under the busy-tracking heuristic; a higher value just
+ /// oversubscribes the queues.
+ UInt32Envar OMPX_TaskGraphLanes;
+
/// Stream manager for AMDGPU streams.
AMDGPUStreamManagerTy AMDGPUStreamManager;
@@ -3885,8 +3957,650 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
/// Tracker for virtual address reservations.
VMemTrackerTy<hsa_amd_vmem_alloc_handle_t> VMemTracker;
+
+ //===----- taskgraph replay backend state --------------------------===//
+ /// The host endpoint of one of the plan's ranges, locked so that a copy over
+ /// it is a single SDMA transfer with no staging buffer. Parallel to
+ /// TaskGraphDataPlanTy::Ranges; left empty for a range no copy touches.
+ struct AmdGraphPinTy {
+ /// Device-accessible alias of the locked host range.
+ void *PinnedHostPtr = nullptr;
+ /// The host address that was locked, kept so it can be unlocked without
+ /// depending on the plan still holding it.
+ void *HostPtr = nullptr;
+ /// We locked the range ourselves and must unlock it. False when
+ /// PinnedHostPtr aliases a registration the program already owned, which
+ /// outlives this graph and must be left alone.
+ bool OwnsPin = false;
+ /// Some copy works over this range, so it is locked for every replay,
+ /// wherever relocation has put it.
+ bool NeedsPin = false;
+ };
+
+ /// One of a leaf's plan operations, with what the lane needs to issue it.
+ /// A leaf's operations run in order on its lane.
+ struct AmdGraphOpTy {
+ /// The plan operation. A copy-like one's operands and enablement are
+ /// read from the plan as it is issued, so a mapping appearing,
+ /// disappearing or moving between replays needs nothing done here.
+ const TaskGraphOpTy *PlanOp = nullptr;
+
+ TaskGraphOpTy::KindTy kind() const { return PlanOp->Kind; }
+
+ //--- Launch ---//
+ const AMDGPUKernelTy *KernelObj = nullptr;
+ /// The kernarg image, complete except that it is copied into a fresh
+ /// device-visible kernarg allocation on each replay.
+ std::vector<uint8_t> KernArgs;
+ /// The geometry, group memory and launch environment, which libomptarget
+ /// calculated for the leaf and cached when it built the plan.
+ TaskGraphLaunchTy Launch;
+ };
+
+ /// A kernel launch whose argument image is written from the plan, naming the
+ /// op it belongs to.
+ struct AmdGraphKernArgsTy {
+ uint32_t Item = 0;
+ uint32_t Op = 0;
+ /// The leaf whose kernel parameters the image holds.
+ const TaskGraphTargetNode *Node = nullptr;
+ };
+
+ /// One graph leaf, as the lowering placed it, with its plan lowered onto
+ /// its lane. Items are numbered in placement order, which is a topological
+ /// order: an item only ever waits for, or follows on its lane, items with
+ /// lower numbers.
+ struct AmdGraphItemTy {
+ /// The leaf; owned by the graph, which outlives this.
+ const TaskGraphNode *Node = nullptr;
+ /// The lane the item is issued onto.
+ uint32_t Lane = 0;
+ /// The items this one must not start before, as the lowering reduced them:
+ /// none on its own lane, and at most one per other lane.
+ SmallVector<uint32_t, 4> Waits;
+ std::vector<AmdGraphOpTy> Ops;
+ /// Non-null iff another item waits on this one; recorded after the item's
+ /// last op on every replay. Owned by the graph.
+ AMDGPUEventTy *Completion = nullptr;
+ };
+
+ /// Everything finalize built for one graph, which libomptarget keeps for us
+ /// in the graph's PluginGraph.
+ struct AmdTaskGraphTy {
+ /// The graph this was built for, whose plan replay reads.
+ const TaskGraphTy *Graph = nullptr;
+ /// Stream lanes acquired at finalize, returned at destroy. One per lane
+ /// the graph was lowered onto, each on its own AQL queue.
+ SmallVector<AMDGPUStreamTy *, 4> Lanes;
+ std::vector<AmdGraphPinTy> Pins;
+ /// The host lock over the plan's attach staging bytes. One lock for all of
+ /// them: the plan lays them end to end in memory it owns for the graph's
+ /// lifetime, so unlike a range this never has to be retaken.
+ AmdGraphPinTy StagingPin;
+ /// The host lock over the plan's private-block image, on the same terms.
+ AmdGraphPinTy PrivatePin;
+ std::vector<AmdGraphItemTy> Items;
+ std::vector<AmdGraphKernArgsTy> KernArgs;
+ /// Per plan argument entry, whether the next replay copies a private's
+ /// staged bytes in: they changed since the device last took them. Set by
+ /// reparametrization and cleared as the copy is issued.
+ std::vector<bool> PrivateInitPending;
+ };
+
+ /// Release everything a finalized graph holds. Written to tolerate a
+ /// partially-built graph so finalize can bail out through it.
+ Error releaseTaskGraph(AmdTaskGraphTy &TG);
+
+ /// Write one kernel launch's argument image against the plan's current
+ /// device addresses.
+ Error materializeKernArgs(const llvm::omp::target::TaskGraphDataPlanTy &Plan,
+ AmdTaskGraphTy &TG, AmdGraphKernArgsTy &Pending);
+
+ /// Lock one host span, releasing whatever \p Pin held before.
+ Error pinTaskGraphSpan(AmdGraphPinTy &Pin, void *HostBegin, uint64_t Size);
+
+ /// Lock one plan range's host endpoint, releasing whatever was locked for it
+ /// before. Re-run for a range relocation has moved.
+ Error pinTaskGraphRange(AmdTaskGraphTy &TG,
+ const llvm::omp::target::TaskGraphRangeTy &Range);
+
+ /// Settle which plan ranges some copy touches, and lock the plan's staging
+ /// bytes, which live as long as the graph does.
+ Error prepareTaskGraphPins(const llvm::omp::target::TaskGraphDataPlanTy &Plan,
+ AmdTaskGraphTy &TG);
+
+ /// Lock the host endpoint of every plan range some copy touches, for one
+ /// replay.
+ Error pinTaskGraphRanges(const llvm::omp::target::TaskGraphDataPlanTy &Plan,
+ AmdTaskGraphTy &TG);
+
+ /// Release what pinTaskGraphRanges locked.
+ Error unpinTaskGraphRanges(AmdTaskGraphTy &TG);
+
+ /// The finalized state for \p Graph, which this device claimed.
+ static AmdTaskGraphTy &
+ getTaskGraph(const llvm::omp::target::TaskGraphTy *Graph) {
+ assert(Graph->PluginGraph && "taskgraph this plugin never claimed");
+ return *static_cast<AmdTaskGraphTy *>(Graph->PluginGraph);
+ }
+
+ /// Drain every lane of a finalized graph.
+ Error syncTaskGraphLanes(AmdTaskGraphTy &TG);
+
+ /// Place leaf \p Node as item \p I of \p TG, and lower its plan operations
+ /// onto its lane. What finalize's lowering does with each leaf it places.
+ Error placeTaskGraphItem(AmdTaskGraphTy &TG, uint32_t I,
+ const TaskGraphNode &Node, uint32_t Lane,
+ ArrayRef<uint32_t> Waits);
+
+ /// Enqueue item \p I's ops onto its lane, behind the items it waits for, and
+ /// record its completion.
+ Error issueTaskGraphItem(AmdTaskGraphTy &TG, uint32_t I, uint64_t StackSize);
};
+Error AMDGPUDeviceTy::releaseTaskGraph(AmdTaskGraphTy &TG) {
+ Error Err = Plugin::success();
+ auto Join = [&Err](Error E) {
+ Err = joinErrors(std::move(Err), std::move(E));
+ };
+
+ for (AmdGraphItemTy &Item : TG.Items) {
+ if (Item.Completion) {
+ Join(AMDGPUEventManager.returnResource(Item.Completion));
+ Item.Completion = nullptr;
+ }
+ }
+ for (AMDGPUStreamTy *Lane : TG.Lanes)
+ if (Lane)
+ Join(AMDGPUStreamManager.returnResource(Lane));
+ TG.Lanes.clear();
+ // The device memory belongs to the plan, which libomptarget releases; only
+ // the host locks are ours.
+ auto Unpin = [&](AmdGraphPinTy &Pin) {
+ if (Pin.PinnedHostPtr && Pin.OwnsPin)
+ Join(Plugin::check(hsa_amd_memory_unlock(Pin.HostPtr),
+ "error unpinning taskgraph host buffer: %s"));
+ Pin.PinnedHostPtr = nullptr;
+ Pin.OwnsPin = false;
+ };
+ for (AmdGraphPinTy &Pin : TG.Pins)
+ Unpin(Pin);
+ Unpin(TG.StagingPin);
+ Unpin(TG.PrivatePin);
+ TG.Pins.clear();
+ return Err;
+}
+
+Error AMDGPUDeviceTy::materializeKernArgs(const TaskGraphDataPlanTy &Plan,
+ AmdTaskGraphTy &TG,
+ AmdGraphKernArgsTy &Pending) {
+ AmdGraphOpTy &Op = TG.Items[Pending.Item].Ops[Pending.Op];
+
+ ArrayRef<TaskGraphKernelParamTy> Params = Pending.Node->kernelParams();
+ SmallVector<uint64_t, 16> Values(Params.size());
+ SmallVector<void *, 16> Ptrs(Params.size());
+ for (auto [P, Param] : llvm::enumerate(Params)) {
+ Values[P] =
+ Plan.paramValue(*Pending.Node, Param, Op.Launch.LaunchEnvironment);
+ Ptrs[P] = &Values[P];
+ }
+
+ Op.KernArgs.assign(Op.KernelObj->getArgsSize(), 0);
+ return Op.KernelObj->writeArgs(Op.KernArgs.data(), Ptrs.size(), Ptrs.data(),
+ Op.Launch.NumThreads, Op.Launch.NumBlocks,
+ Op.Launch.DynBlockMemSize);
+}
+
+Error AMDGPUDeviceTy::pinTaskGraphSpan(AmdGraphPinTy &Pin, void *HostBegin,
+ uint64_t Size) {
+ // Relocation can move a range between replays, and the lock names an
+ // address, so release the one taken against where the range used to be.
+ if (Pin.PinnedHostPtr && Pin.OwnsPin)
+ if (auto Err = Plugin::check(hsa_amd_memory_unlock(Pin.HostPtr),
+ "error unpinning taskgraph host buffer: %s"))
+ return Err;
+ Pin.PinnedHostPtr = nullptr;
+ Pin.HostPtr = nullptr;
+ Pin.OwnsPin = false;
+
+ if (!HostBegin || !Size)
+ return Plugin::error(ErrorCode::UNSUPPORTED,
+ "amdgpu taskgraph: copy against a region with no "
+ "host address");
+
+ // A range HSA already knows about must not be locked a second time: HSA
+ // rejects the duplicate lock, and taking that as a hard failure would
+ // decline graphs we can serve perfectly well (any program mapping buffers
+ // obtained from llvm_omp_target_alloc_host hits this on every graph).
+ // Reuse the registration's own device-accessible alias instead, and
+ // remember that it is not ours to release.
+ void *BaseHostPtr = nullptr, *BaseAgentPtr = nullptr;
+ size_t BaseSize = 0;
+ auto Registered =
+ isPinnedPtrImpl(HostBegin, BaseHostPtr, BaseAgentPtr, BaseSize);
+ if (!Registered)
+ return Registered.takeError();
+
+ Pin.HostPtr = HostBegin;
+ if (*Registered) {
+ // The alias only describes the bytes the existing registration covers, so
+ // a range running past its end has no single device-accessible address
+ // and cannot be served as one transfer.
+ const char *Base = reinterpret_cast<const char *>(BaseHostPtr);
+ const char *Host = reinterpret_cast<const char *>(HostBegin);
+ if (Host < Base || static_cast<uint64_t>(Host - Base) + Size > BaseSize)
+ return Plugin::error(ErrorCode::UNSUPPORTED,
+ "amdgpu taskgraph: copy against a partially "
+ "pinned host range");
+ Pin.PinnedHostPtr = reinterpret_cast<char *>(BaseAgentPtr) + (Host - Base);
+ Pin.OwnsPin = false;
+ return Plugin::success();
+ }
+
+ void *Pinned = nullptr;
+ if (auto Err = Plugin::check(
+ hsa_amd_memory_lock(HostBegin, Size, nullptr, 0, &Pinned),
+ "error pinning taskgraph host buffer: %s"))
+ return Err;
+ Pin.PinnedHostPtr = Pinned;
+ Pin.OwnsPin = true;
+ return Plugin::success();
+}
+
+Error AMDGPUDeviceTy::pinTaskGraphRange(AmdTaskGraphTy &TG,
+ const TaskGraphRangeTy &Range) {
+ return pinTaskGraphSpan(TG.Pins[Range.getId()], Range.getHostBegin(),
+ Range.getSize());
+}
+
+Error AMDGPUDeviceTy::pinTaskGraphRanges(const TaskGraphDataPlanTy &Plan,
+ AmdTaskGraphTy &TG) {
+ for (const TaskGraphRangeTy &Range : Plan.ranges())
+ if (TG.Pins[Range.getId()].NeedsPin)
+ if (auto Err = pinTaskGraphRange(TG, Range))
+ return Err;
+ return Plugin::success();
+}
+
+Error AMDGPUDeviceTy::unpinTaskGraphRanges(AmdTaskGraphTy &TG) {
+ Error Err = Plugin::success();
+ for (AmdGraphPinTy &Pin : TG.Pins) {
+ if (Pin.PinnedHostPtr && Pin.OwnsPin)
+ Err = joinErrors(
+ std::move(Err),
+ Plugin::check(hsa_amd_memory_unlock(Pin.HostPtr),
+ "error unpinning taskgraph host buffer: %s"));
+ Pin.PinnedHostPtr = nullptr;
+ Pin.HostPtr = nullptr;
+ Pin.OwnsPin = false;
+ }
+ return Err;
+}
+
+Error AMDGPUDeviceTy::prepareTaskGraphPins(const TaskGraphDataPlanTy &Plan,
+ AmdTaskGraphTy &TG) {
+ // Every range some copy touches is locked for each replay, whether or not
+ // the copy is enabled for it, since the plan may switch it on without the
+ // lock being retaken. The lock itself is per replay: the plan's ranges are
+ // only known to be the program's memory while a replay has them resolved.
+ TG.Pins.assign(Plan.ranges().size(), AmdGraphPinTy{});
+ for (const AmdGraphItemTy &Item : TG.Items)
+ for (const TaskGraphOpTy &Op : Item.Node->ops())
+ if (Op.Kind == TaskGraphOpTy::KindTy::CopyIn ||
+ Op.Kind == TaskGraphOpTy::KindTy::CopyOut)
+ TG.Pins[Plan.transfer(*Item.Node, Op).HostRange->getId()].NeedsPin =
+ true;
+
+ // The staging bytes are the source of every attachment's copy, and they live
+ // in the plan for as long as the graph does, so one lock over the lot of them
+ // serves every replay.
+ if (!Plan.attachStaging().empty()) {
+ TG.StagingPin.NeedsPin = true;
+ if (auto Err = pinTaskGraphSpan(
+ TG.StagingPin, const_cast<char *>(Plan.attachStaging().data()),
+ Plan.attachStaging().size()))
+ return Err;
+ }
+ // Likewise the host image of the private block, for the same reason and with
+ // the same lifetime.
+ if (!Plan.privateStaging().empty()) {
+ TG.PrivatePin.NeedsPin = true;
+ if (auto Err = pinTaskGraphSpan(
+ TG.PrivatePin, const_cast<char *>(Plan.privateStaging().data()),
+ Plan.privateStaging().size()))
+ return Err;
+ }
+ return Plugin::success();
+}
+
+Error AMDGPUDeviceTy::placeTaskGraphItem(AmdTaskGraphTy &TG, uint32_t I,
+ const TaskGraphNode &Node,
+ uint32_t Lane,
+ ArrayRef<uint32_t> Waits) {
+ const TaskGraphTy &Graph = *TG.Graph;
+ auto decline = [](const char *Why) -> Error {
+ return Plugin::error(ErrorCode::UNSUPPORTED,
+ "amdgpu taskgraph declined: %s", Why);
+ };
+
+ assert(I == TG.Items.size() && "taskgraph items placed out of order");
+ TG.Items.emplace_back();
+ AmdGraphItemTy &Item = TG.Items.back();
+ Item.Node = &Node;
+ Item.Lane = Lane;
+ Item.Waits.assign(Waits.begin(), Waits.end());
+
+ // A wait for an item is a wait for its completion event, which is recorded
+ // only for an item somebody waits for.
+ for (uint32_t Wait : Waits) {
+ AmdGraphItemTy &Awaited = TG.Items[Wait];
+ if (!Awaited.Completion)
+ if (auto Err = AMDGPUEventManager.getResource(Awaited.Completion))
+ return Err;
+ }
+
+ // Lower the leaf's plan operations onto its lane, in order. Every decision
+ // about mapped data -- which ranges there are, where each lives on the
+ // device, which transfers a replay performs, what each kernel parameter is
+ // -- has already been made by the plan, so nothing below classifies a map or
+ // allocates device data.
+ for (const TaskGraphOpTy &PlanOp : Node.ops()) {
+ AmdGraphOpTy Op;
+ Op.PlanOp = &PlanOp;
+
+ if (PlanOp.Kind == TaskGraphOpTy::KindTy::Launch) {
+ auto &TargetNode = cast<TaskGraphTargetNode>(Node);
+ void *Resolved =
+ Graph.Services.resolveKernel(Graph.DeviceId, TargetNode.HostPtr);
+ if (!Resolved)
+ return decline("target leaf's kernel is not loaded on this device");
+ Op.KernelObj = static_cast<AMDGPUKernelTy *>(
+ static_cast<GenericKernelTy *>(Resolved));
+ // libomptarget calculated the launch once and cached it, where a launch
+ // outside a graph works it out every time; its device memory belongs to
+ // the plan.
+ Op.Launch = TargetNode.launch();
+ TG.KernArgs.push_back(
+ {I, static_cast<uint32_t>(Item.Ops.size()), &TargetNode});
+ }
+ Item.Ops.push_back(std::move(Op));
+ }
+ return Plugin::success();
+}
+
+Expected<void *> AMDGPUDeviceTy::finalizeTaskGraph(const TaskGraphTy *Graph) {
+
+ // supportsTaskGraphs is what libomptarget checks before it builds a plan or
+ // offers a graph to anybody, so a backend switched off there is never asked
+ // to finalize one.
+ auto Owned = std::make_unique<AmdTaskGraphTy>();
+ AmdTaskGraphTy &TG = *Owned;
+ TG.Graph = Graph;
+
+ // Unwind whatever has been built so far and propagate \p Reason. Declining
+ // and failing take the same route out: to libomp both mean "this plugin will
+ // not be replaying the graph", and the graph itself is unharmed either way.
+ auto bail = [&](Error Reason) -> Error {
+ return joinErrors(std::move(Reason), releaseTaskGraph(TG));
+ };
+ auto decline = [&](const char *Why) -> Error {
+ return bail(Plugin::error(ErrorCode::UNSUPPORTED,
+ "amdgpu taskgraph declined: %s", Why));
+ };
+
+ // 1. Lanes. Each lands on its own AQL queue, which is what makes the
+ // schedule's concurrency real and what keeps replay deadlock-free: a lane
+ // that runs its ring full only throttles the host, whereas two lanes sharing
+ // a ring could put a cross-lane barrier in front of the packet that satisfies
+ // it. Ask for as many as there are queues unless overridden; the number we
+ // actually get is the lane count the schedule is built for.
+ const uint32_t WantLanes =
+ std::max<uint32_t>(1, OMPX_TaskGraphLanes ? OMPX_TaskGraphLanes.get()
+ : OMPX_NumQueues.get());
+ if (auto Err =
+ AMDGPUStreamManager.getDistinctQueueResources(WantLanes, TG.Lanes))
+ return bail(std::move(Err));
+ if (TG.Lanes.empty())
+ return decline("no stream lanes available");
+
+ // 2. Lower the region tree onto those lanes. Each leaf is built into an
+ // item the moment the lowering has placed it, which it does in an order
+ // where everything a leaf waits for is placed first.
+ struct SinkTy final : llvm::omp::target::plugin::TaskGraphLaneSinkTy {
+ SinkTy(AMDGPUDeviceTy &Device, AmdTaskGraphTy &TG)
+ : Device(Device), TG(TG) {}
+
+ Error place(uint32_t Item, const TaskGraphNode &Leaf, uint32_t Lane,
+ ArrayRef<uint32_t> Waits) override {
+ return Device.placeTaskGraphItem(TG, Item, Leaf, Lane, Waits);
+ }
+
+ AMDGPUDeviceTy &Device;
+ AmdTaskGraphTy &TG;
+ } Sink(*this, TG);
+ if (auto Err = llvm::omp::target::plugin::lowerTaskGraphOntoLanes(
+ *Graph, TG.Lanes.size(), Sink))
+ return bail(std::move(Err));
+
+ // 3. Note the host endpoint of every range a copy touches, which each replay
+ // pins so that each copy is a single SDMA transfer against the host range
+ // with no staging buffer.
+ const TaskGraphDataPlanTy &Plan = Graph->Plan;
+ if (auto Err = prepareTaskGraphPins(Plan, TG))
+ return bail(std::move(Err));
+ TG.PrivateInitPending.assign(Plan.getNumArgs(), false);
+
+ // 4. Write the kernarg images against the plan's current device addresses.
+ for (AmdGraphKernArgsTy &Pending : TG.KernArgs)
+ if (auto Err = materializeKernArgs(Plan, TG, Pending))
+ return bail(std::move(Err));
+
+ ODBG(OLDT_Module) << "amdgpu taskgraph " << Graph << ": " << TG.Items.size()
+ << " items on " << TG.Lanes.size()
+ << " lanes (one AQL queue each), "
+ << llvm::sum_of(
+ llvm::map_range(TG.Items,
+ [](const AmdGraphItemTy &Item) {
+ return Item.Waits.size();
+ }))
+ << " waits, " << Plan.ranges().size() << " mapped ranges";
+
+ return Owned.release();
+}
+
+Error AMDGPUDeviceTy::syncTaskGraphLanes(AmdTaskGraphTy &TG) {
+ Error Err = Plugin::success();
+ for (AMDGPUStreamTy *Lane : TG.Lanes)
+ Err = joinErrors(std::move(Err), Lane->synchronize());
+ return Err;
+}
+
+Error AMDGPUDeviceTy::issueTaskGraphItem(AmdTaskGraphTy &TG, uint32_t I,
+ uint64_t StackSize) {
+ const TaskGraphDataPlanTy &Plan = TG.Graph->Plan;
+ AMDGPUMemoryManagerTy &ArgsMemoryManager = HostDevice.getArgsMemoryManager();
+ AmdGraphItemTy &Item = TG.Items[I];
+
+ AMDGPUStreamTy *Lane = TG.Lanes[Item.Lane];
+
+ // Gate on the other lanes this item depends on. One barrier covers them
+ // all; ordering against earlier work on this lane is implicit. Items are
+ // issued in placement order, so each event named here has already been
+ // recorded by this replay.
+ SmallVector<AMDGPUEventTy *, 4> Events;
+ for (uint32_t Wait : Item.Waits)
+ Events.push_back(TG.Items[Wait].Completion);
+ if (!Events.empty())
+ if (auto Err = Lane->waitEvents(Events))
+ return Err;
+
+ // The pinned alias of a transfer's host end. Each alias covers its buffer
+ // from the start, so it takes the same offset into it as the plan's address.
+ const TaskGraphNode &Leaf = *Item.Node;
+ auto pinnedHostFor = [&](const TaskGraphOpTy &PlanOp,
+ const TaskGraphTransferTy &T) -> char * {
+ const char *Host = static_cast<const char *>(T.ToDevice ? T.Src : T.Dst);
+ auto alias = [&](const AmdGraphPinTy &Pin, const char *Base) {
+ return static_cast<char *>(Pin.PinnedHostPtr) + (Host - Base);
+ };
+ switch (PlanOp.Kind) {
+ case TaskGraphOpTy::KindTy::Attach:
+ return alias(TG.StagingPin, Plan.attachStaging().data());
+ case TaskGraphOpTy::KindTy::PrivateInit:
+ return alias(TG.PrivatePin, Plan.privateStaging().data());
+ default:
+ return alias(TG.Pins[T.HostRange->getId()], T.HostRange->begin());
+ }
+ };
+
+ for (AmdGraphOpTy &Op : Item.Ops) {
+ if (Op.kind() == TaskGraphOpTy::KindTy::Launch) {
+ // Per-replay kernarg buffer, released by the stream once the dispatch
+ // completes (schedReleaseBuffer in pushKernelLaunch).
+ void *Args = nullptr;
+ if (auto Err = ArgsMemoryManager.allocate(Op.KernArgs.size(), &Args))
+ return Err;
+ std::memcpy(Args, Op.KernArgs.data(), Op.KernArgs.size());
+ // HSA's group segment size covers both static and dynamic memory.
+ if (auto Err = Lane->pushKernelLaunch(
+ *Op.KernelObj, Args, Op.Launch.NumThreads, Op.Launch.NumBlocks,
+ Op.KernelObj->getStaticBlockMemSize() + Op.Launch.DynBlockMemSize,
+ StackSize, ArgsMemoryManager))
+ return Err;
+ continue;
+ }
+
+ // A transfer this replay does not perform: a range that was already
+ // mapped when the replay began has its implicit transfers performed by
+ // whoever brought it into the device data environment, and an attachment
+ // with either end unmapped or host-resident has no device copy of the
+ // pointer to correct.
+ TaskGraphTransferTy T = Plan.transfer(Leaf, *Op.PlanOp);
+ if (!T.Enabled)
+ continue;
+ // Nothing on the device writes the private block, so a private whose
+ // staged bytes have not changed is already in place.
+ if (Op.kind() == TaskGraphOpTy::KindTy::PrivateInit) {
+ const uint32_t Index = Plan.argIndex(Leaf.argPlan()[Op.PlanOp->Arg]);
+ if (!TG.PrivateInitPending[Index])
+ continue;
+ TG.PrivateInitPending[Index] = false;
+ }
+ char *Host = pinnedHostFor(*Op.PlanOp, T);
+ if (auto Err = Lane->pushPinnedMemoryCopyAsync(
+ T.ToDevice ? T.Dst : Host, T.ToDevice ? Host : T.Src, T.Size))
+ return Err;
+ }
+
+ if (Item.Completion)
+ if (auto Err = Item.Completion->record(*Lane))
+ return Err;
+ return Plugin::success();
+}
+
+Error AMDGPUDeviceTy::replayTaskGraph(const TaskGraphTy *Graph) {
+ AmdTaskGraphTy &TG = getTaskGraph(Graph);
+
+ // Private segment size for kernels with a dynamically-sized stack (mirrors
+ // launchImpl; pushKernelLaunch picks this or the kernel's static private size
+ // depending on usesDynamicStack()).
+ uint64_t StackSize = 0;
+ if (auto Err = getDeviceStackSize(StackSize))
+ return Err;
+
+ // Everything is issued without waiting on the device, so a lane can enqueue
+ // more packets than its AQL ring holds, at which point acquirePacket
+ // throttles the host until the command processor drains older ones. That is
+ // back-pressure, not deadlock, and only because of the two properties below:
+ // a lane owns its ring (getDistinctQueueResources), and items are issued in
+ // placement order, so every item an item waits for is issued before it.
+ // Every dependency of a packet is therefore already enqueued when the barrier
+ // naming it is written, and the earliest incomplete packet on any lane always
+ // has all of its dependencies ahead of it -- the queues always drain. Two
+ // lanes sharing one ring is what would break this: a barrier could then sit
+ // in front of the very packet that satisfies it.
+ //
+ // Copies need the same argument made about a ring no lane can own. Every
+ // copy on the device goes through one in-order blit queue shared with the
+ // rest of the process, and a copy waiting on a dependency signal stalls its
+ // head, so a later copy cannot overtake it however unrelated it is. Issue
+ // order saves this too: a gated copy only ever names work already enqueued,
+ // so it clears on its own, and whatever queues behind it waits only for that.
+ //
+ // The host endpoints were locked for this replay by pinTaskGraph, and the
+ // kernarg images are stable unless the mapping state moved under the graph --
+ // in which case libomptarget has already refreshed the plan and had us adopt
+ // it through reparametrizeTaskGraph.
+ Error Err = Plugin::success();
+ for (uint32_t I = 0; I < TG.Items.size() && !Err; ++I)
+ if ((Err = issueTaskGraphItem(TG, I, StackSize)))
+ break;
+
+ // Drained even after a failure, so that nothing issued is still running when
+ // libomptarget releases the graph's mappings.
+ return joinErrors(std::move(Err), syncTaskGraphLanes(TG));
+}
+
+Error AMDGPUDeviceTy::reparametrizeTaskGraph(
+ const TaskGraphTy *Graph, ArrayRef<TaskGraphArgUpdateTy> Updates,
+ bool Resync) {
+ AmdTaskGraphTy &TG = getTaskGraph(Graph);
+ const TaskGraphDataPlanTy &Plan = Graph->Plan;
+
+ // A copy-like operation reads its addresses, enablement and staged bytes out
+ // of the plan as it is issued, and its host endpoint was locked where it now
+ // is by pinTaskGraph, so it needs nothing done here.
+ SmallVector<uint8_t, 16> Changes(
+ Plan.getNumArgs(), Resync ? TaskGraphArgUpdateTy::AllChanged : 0);
+ for (const TaskGraphArgUpdateTy &Update : Updates)
+ Changes[Update.Arg] |= Update.Changes;
+
+ // Except that a private whose bytes have not changed need not be copied in
+ // again.
+ TG.PrivateInitPending.resize(Plan.getNumArgs());
+ for (uint32_t Index = 0; Index < Plan.getNumArgs(); ++Index)
+ if (isa<TaskGraphFirstPrivateArgTy>(Plan.getArg(Index).get()) &&
+ (Changes[Index] & TaskGraphArgUpdateTy::DataChanged))
+ TG.PrivateInitPending[Index] = true;
+
+ // A kernel's arguments, in contrast, are an image written once, so one has
+ // to be rewritten when a parameter's address or value changed. The image is
+ // host memory copied into a fresh kernarg allocation at each launch, so
+ // rewriting it involves no device copy.
+ constexpr uint8_t Rewrite = TaskGraphArgUpdateTy::DeviceAddressChanged |
+ TaskGraphArgUpdateTy::DataChanged;
+ for (AmdGraphKernArgsTy &Pending : TG.KernArgs)
+ if (llvm::any_of(
+ Pending.Node->kernelParams(), [&](const TaskGraphKernelParamTy &P) {
+ return P.Kind !=
+ TaskGraphKernelParamTy::KindTy::LaunchEnvironment &&
+ (Changes[Plan.argIndex(Pending.Node->argPlan()[P.Arg])] &
+ Rewrite);
+ }))
+ if (auto Err = materializeKernArgs(Plan, TG, Pending))
+ return Err;
+ return Plugin::success();
+}
+
+Error AMDGPUDeviceTy::pinTaskGraph(const TaskGraphTy *Graph) {
+ AmdTaskGraphTy &TG = getTaskGraph(Graph);
+ // Every range, every replay, wherever relocation has put it: nothing yet
+ // says which ranges are sure to outlive the replay.
+ if (auto Err = pinTaskGraphRanges(Graph->Plan, TG))
+ return joinErrors(std::move(Err), unpinTaskGraphRanges(TG));
+ return Plugin::success();
+}
+
+Error AMDGPUDeviceTy::unpinTaskGraph(const TaskGraphTy *Graph) {
+ AmdTaskGraphTy &TG = getTaskGraph(Graph);
+ return unpinTaskGraphRanges(TG);
+}
+
+Error AMDGPUDeviceTy::destroyTaskGraph(const TaskGraphTy *Graph) {
+ std::unique_ptr<AmdTaskGraphTy> TG(&getTaskGraph(Graph));
+ return releaseTaskGraph(*TG);
+}
+
Error AMDGPUDeviceImageTy::loadExecutable(const AMDGPUDeviceTy &Device) {
hsa_code_object_reader_t Reader;
hsa_status_t Status =
@@ -4535,6 +5249,68 @@ AMDGPUPluginContextTy::getAllocInfo(const void *Ptr) {
HsaInfo.sizeInBytes};
}
+Error AMDGPUKernelTy::writeArgs(void *Buf, uint32_t NumArgs, void *const *Args,
+ const uint32_t NumThreads[3],
+ const uint32_t NumBlocks[3],
+ uint32_t DynBlockMemSize) const {
+ // Copy explicit arguments.
+ size_t ExplicitEnd = 0;
+ if (NumArgs) {
+ const auto &ArgMDs = KernelInfo.ArgMDs;
+ if (NumArgs > ArgMDs.size())
+ return Plugin::error(
+ ErrorCode::INVALID_ARGUMENT,
+ "number of arguments (%u) exceeds the number of arguments "
+ "expected by the kernel (%zu)",
+ NumArgs, ArgMDs.size());
+
+ for (size_t I = 0; I < NumArgs; I++) {
+ auto [Offset, Size] = ArgMDs[I];
+ std::memcpy(utils::advancePtr(Buf, Offset), Args[I], Size);
+ }
+
+ auto [Offset, Size] = ArgMDs[NumArgs - 1];
+ ExplicitEnd = Offset + Size;
+ }
+
+ uint64_t ImplArgsOffset =
+ llvm::alignTo(ExplicitEnd, alignof(hsa_utils::AMDGPUImplicitArgsTy));
+ if (ArgsSize <= ImplArgsOffset)
+ return Plugin::success();
+
+ hsa_utils::AMDGPUImplicitArgsTy *ImplArgs =
+ reinterpret_cast<hsa_utils::AMDGPUImplicitArgsTy *>(
+ utils::advancePtr(Buf, ImplArgsOffset));
+
+ // Set the COV5+ implicit arguments to the appropriate values if present.
+ uint64_t ImplArgsSize = ArgsSize - ImplArgsOffset;
+ std::memset(ImplArgs, 0, ImplArgsSize);
+
+ using ImplArgsTy = hsa_utils::AMDGPUImplicitArgsTy;
+ hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::BlockCountX, ImplArgsSize,
+ NumBlocks[0]);
+ hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::BlockCountY, ImplArgsSize,
+ NumBlocks[1]);
+ hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::BlockCountZ, ImplArgsSize,
+ NumBlocks[2]);
+
+ hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GroupSizeX, ImplArgsSize,
+ NumThreads[0]);
+ hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GroupSizeY, ImplArgsSize,
+ NumThreads[1]);
+ hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GroupSizeZ, ImplArgsSize,
+ NumThreads[2]);
+
+ hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GridDims, ImplArgsSize,
+ NumBlocks[2] * NumThreads[2] > 1
+ ? 3
+ : 1 + (NumBlocks[1] * NumThreads[1] != 1));
+
+ hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::DynamicLdsSize, ImplArgsSize,
+ DynBlockMemSize);
+ return Plugin::success();
+}
+
Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
uint32_t NumThreads[3], uint32_t NumBlocks[3],
uint32_t DynBlockMemSize,
@@ -4557,67 +5333,15 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
if (auto Err = GenericDevice.getDeviceStackSize(StackSize))
return Err;
- // Copy explicit arguments.
- size_t ExplicitEnd = 0;
- if (LaunchArgs.Args) {
- const auto &ArgMDs = KernelInfo.ArgMDs;
- uint32_t NumArgs = LaunchArgs.NumArgs;
-
- if (NumArgs > ArgMDs.size())
- return Plugin::error(
- ErrorCode::INVALID_ARGUMENT,
- "number of arguments (%u) exceeds the number of arguments "
- "expected by the kernel (%zu)",
- NumArgs, ArgMDs.size());
-
- for (size_t I = 0; I < NumArgs; I++) {
- auto [Offset, Size] = ArgMDs[I];
- std::memcpy(utils::advancePtr(AllArgs, Offset), LaunchArgs.Args[I], Size);
- }
-
- auto [Offset, Size] = ArgMDs[NumArgs - 1];
- ExplicitEnd = Offset + Size;
- }
+ if (auto Err =
+ writeArgs(AllArgs, LaunchArgs.Args ? LaunchArgs.NumArgs : 0,
+ LaunchArgs.Args, NumThreads, NumBlocks, DynBlockMemSize))
+ return Err;
AMDGPUStreamTy *Stream = nullptr;
if (auto Err = AMDGPUDevice.getStream(AsyncInfoWrapper, Stream))
return Err;
- uint64_t ImplArgsOffset =
- llvm::alignTo(ExplicitEnd, alignof(hsa_utils::AMDGPUImplicitArgsTy));
- if (ArgsSize > ImplArgsOffset) {
- hsa_utils::AMDGPUImplicitArgsTy *ImplArgs =
- reinterpret_cast<hsa_utils::AMDGPUImplicitArgsTy *>(
- utils::advancePtr(AllArgs, ImplArgsOffset));
-
- // Set the COV5+ implicit arguments to the appropriate values if present.
- uint64_t ImplArgsSize = ArgsSize - ImplArgsOffset;
- std::memset(ImplArgs, 0, ImplArgsSize);
-
- using ImplArgsTy = hsa_utils::AMDGPUImplicitArgsTy;
- hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::BlockCountX, ImplArgsSize,
- NumBlocks[0]);
- hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::BlockCountY, ImplArgsSize,
- NumBlocks[1]);
- hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::BlockCountZ, ImplArgsSize,
- NumBlocks[2]);
-
- hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GroupSizeX, ImplArgsSize,
- NumThreads[0]);
- hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GroupSizeY, ImplArgsSize,
- NumThreads[1]);
- hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GroupSizeZ, ImplArgsSize,
- NumThreads[2]);
-
- hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::GridDims, ImplArgsSize,
- NumBlocks[2] * NumThreads[2] > 1
- ? 3
- : 1 + (NumBlocks[1] * NumThreads[1] != 1));
-
- hsa_utils::initImplArg(ImplArgs, &ImplArgsTy::DynamicLdsSize, ImplArgsSize,
- DynBlockMemSize);
- }
-
// HSA requires the group segment size to include both static and dynamic.
uint32_t TotalBlockMemSize = getStaticBlockMemSize() + DynBlockMemSize;
diff --git a/offload/test/offloading/taskgraph_claim_finalize_partial.c b/offload/test/offloading/taskgraph_claim_finalize_partial.c
index e836215fe9230f..572e97bcdaa4c8 100644
--- a/offload/test/offloading/taskgraph_claim_finalize_partial.c
+++ b/offload/test/offloading/taskgraph_claim_finalize_partial.c
@@ -7,9 +7,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A taskgraph region that is only partly mapped outside the graph when the
// graph is finalized. Whether the graph keeps a region to itself or borrows
diff --git a/offload/test/offloading/taskgraph_decline_info.c b/offload/test/offloading/taskgraph_decline_info.c
index 067581eaff3cdf..32b1e40851cfe4 100644
--- a/offload/test/offloading/taskgraph_decline_info.c
+++ b/offload/test/offloading/taskgraph_decline_info.c
@@ -5,9 +5,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A graph mapping something the data plan cannot express is replayed by libomp
// on the host instead, which is correct but gives up the device graph. Each
diff --git a/offload/test/offloading/taskgraph_exit_data_borrowed.c b/offload/test/offloading/taskgraph_exit_data_borrowed.c
index 5d16e19db98c6e..94b6fd1b614e22 100644
--- a/offload/test/offloading/taskgraph_exit_data_borrowed.c
+++ b/offload/test/offloading/taskgraph_exit_data_borrowed.c
@@ -9,9 +9,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A `target exit data` in a graph that drops a reference the graph did not
// take. Outside a graph the decrement that removes a mapping is the one that
diff --git a/offload/test/offloading/taskgraph_firstprivate_inner.c b/offload/test/offloading/taskgraph_firstprivate_inner.c
index 886b1f09d7194d..8fcc944334bb16 100644
--- a/offload/test/offloading/taskgraph_firstprivate_inner.c
+++ b/offload/test/offloading/taskgraph_firstprivate_inner.c
@@ -8,9 +8,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A replay starts from the taskgraph construct's enclosing data environment,
// and a variable declared inside the region is not part of it: the code that
diff --git a/offload/test/offloading/taskgraph_firstprivate_restage.c b/offload/test/offloading/taskgraph_firstprivate_restage.c
index de6cfaa2acb102..951a3fef3c779d 100644
--- a/offload/test/offloading/taskgraph_firstprivate_restage.c
+++ b/offload/test/offloading/taskgraph_firstprivate_restage.c
@@ -8,9 +8,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A graph copies a firstprivate aggregate into its private storage only when
// the value it is initialised from has changed since the device last saw it,
diff --git a/offload/test/offloading/taskgraph_firstprivate_scalar.c b/offload/test/offloading/taskgraph_firstprivate_scalar.c
index 4e61f01fa76b35..819076863104d3 100644
--- a/offload/test/offloading/taskgraph_firstprivate_scalar.c
+++ b/offload/test/offloading/taskgraph_firstprivate_scalar.c
@@ -8,9 +8,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A firstprivate scalar is passed to the kernel by value. When its variable is
// in the taskgraph construct's enclosing data environment, each replay takes
diff --git a/offload/test/offloading/taskgraph_firstprivate_scalar_reparam.c b/offload/test/offloading/taskgraph_firstprivate_scalar_reparam.c
index 5fdc915f6b7ca7..301830df50eb21 100644
--- a/offload/test/offloading/taskgraph_firstprivate_scalar_reparam.c
+++ b/offload/test/offloading/taskgraph_firstprivate_scalar_reparam.c
@@ -6,9 +6,6 @@
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
// REQUIRES: libomptarget-debug
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// Refreshing a by-value firstprivate costs a kernel-argument rewrite only when
// the value actually changed: a replay passing the value the previous one did
diff --git a/offload/test/offloading/taskgraph_host_task_subgraph_order.c b/offload/test/offloading/taskgraph_host_task_subgraph_order.c
index 2d6c0e910849b2..99d5cb7dc39707 100644
--- a/offload/test/offloading/taskgraph_host_task_subgraph_order.c
+++ b/offload/test/offloading/taskgraph_host_task_subgraph_order.c
@@ -11,9 +11,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// Host tasks that call into the offload runtime themselves, ordered against
// target constructs in the same graph. Host tasks never enter libomptarget:
diff --git a/offload/test/offloading/taskgraph_lambda_capture.cpp b/offload/test/offloading/taskgraph_lambda_capture.cpp
index 16959030e13482..1b7f37cba77fc7 100644
--- a/offload/test/offloading/taskgraph_lambda_capture.cpp
+++ b/offload/test/offloading/taskgraph_lambda_capture.cpp
@@ -5,9 +5,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A lambda capturing by reference, called in a target region, is mapped as the
// lambda object plus one fix-up entry per capture: the runtime writes the
diff --git a/offload/test/offloading/taskgraph_map_drift.c b/offload/test/offloading/taskgraph_map_drift.c
index b51e35919f3083..e52104628befb9 100644
--- a/offload/test/offloading/taskgraph_map_drift.c
+++ b/offload/test/offloading/taskgraph_map_drift.c
@@ -12,9 +12,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A graph is recorded once and replayed many times, but the mapping state it
// was recorded against belongs to the program, not to the graph, and can differ
diff --git a/offload/test/offloading/taskgraph_map_dynamic_bound_saved.c b/offload/test/offloading/taskgraph_map_dynamic_bound_saved.c
index f287e3f6cace92..228494a2976b99 100644
--- a/offload/test/offloading/taskgraph_map_dynamic_bound_saved.c
+++ b/offload/test/offloading/taskgraph_map_dynamic_bound_saved.c
@@ -10,9 +10,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// Two things that pull in opposite directions, in one region.
//
diff --git a/offload/test/offloading/taskgraph_map_partial_borrow.c b/offload/test/offloading/taskgraph_map_partial_borrow.c
index 6d0044ce45ce53..e4c897e18c50c6 100644
--- a/offload/test/offloading/taskgraph_map_partial_borrow.c
+++ b/offload/test/offloading/taskgraph_map_partial_borrow.c
@@ -9,9 +9,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A mapped range inside a taskgraph that only *partially* overlaps one mapped
// outside it. Asking whether a range is mapped is a question about whole-range
diff --git a/offload/test/offloading/taskgraph_map_relocated_mixed.c b/offload/test/offloading/taskgraph_map_relocated_mixed.c
index 2e7bd302e58362..2a6d72121ffe9f 100644
--- a/offload/test/offloading/taskgraph_map_relocated_mixed.c
+++ b/offload/test/offloading/taskgraph_map_relocated_mixed.c
@@ -30,9 +30,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A graph's ranges follow its captures when a replay relocates them, and they
// need not all move by one distance: a pointer parameter's section moves to
diff --git a/offload/test/offloading/taskgraph_map_relocated_stack.c b/offload/test/offloading/taskgraph_map_relocated_stack.c
index 56e63e020b8206..43abb8f2e7bb1a 100644
--- a/offload/test/offloading/taskgraph_map_relocated_stack.c
+++ b/offload/test/offloading/taskgraph_map_relocated_stack.c
@@ -10,9 +10,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A recorded target region keeps the map arrays it was launched with, so the
// host addresses in them are the ones the mapped variables had on the
diff --git a/offload/test/offloading/taskgraph_plan_dump_attach.c b/offload/test/offloading/taskgraph_plan_dump_attach.c
index 992bdd76bb5801..e102680aa9cf4f 100644
--- a/offload/test/offloading/taskgraph_plan_dump_attach.c
+++ b/offload/test/offloading/taskgraph_plan_dump_attach.c
@@ -5,9 +5,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// The data plan of a graph mapping a section through a struct member pointer,
// as dumped before each replay.
diff --git a/offload/test/offloading/taskgraph_plan_dump_borrowed.c b/offload/test/offloading/taskgraph_plan_dump_borrowed.c
index 7545c5e340add9..068ae76df8ce93 100644
--- a/offload/test/offloading/taskgraph_plan_dump_borrowed.c
+++ b/offload/test/offloading/taskgraph_plan_dump_borrowed.c
@@ -5,9 +5,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// The data plan of a graph whose leaves exercise the remaining argument
// properties, as dumped before each replay.
diff --git a/offload/test/offloading/taskgraph_plan_dump_enter_exit.c b/offload/test/offloading/taskgraph_plan_dump_enter_exit.c
index 60798389fe16d1..2cc48ed4858a4c 100644
--- a/offload/test/offloading/taskgraph_plan_dump_enter_exit.c
+++ b/offload/test/offloading/taskgraph_plan_dump_enter_exit.c
@@ -5,9 +5,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// The data plan of a graph that maps its data with enter and exit data around a
// target region, as dumped before each replay.
diff --git a/offload/test/offloading/taskgraph_plan_dump_firstprivate.c b/offload/test/offloading/taskgraph_plan_dump_firstprivate.c
index d8506506dc5c5b..fc15cb205f2a98 100644
--- a/offload/test/offloading/taskgraph_plan_dump_firstprivate.c
+++ b/offload/test/offloading/taskgraph_plan_dump_firstprivate.c
@@ -5,9 +5,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// The data plan of a graph with privatised and by-value arguments, as dumped
// before each replay.
diff --git a/offload/test/offloading/taskgraph_plan_dump_irreducible.c b/offload/test/offloading/taskgraph_plan_dump_irreducible.c
index c7268298af4786..115e94a074c5a9 100644
--- a/offload/test/offloading/taskgraph_plan_dump_irreducible.c
+++ b/offload/test/offloading/taskgraph_plan_dump_irreducible.c
@@ -5,9 +5,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// The data plan of a graph whose dependences are not series-parallel, as dumped
// before each replay.
diff --git a/offload/test/offloading/taskgraph_present_absent.c b/offload/test/offloading/taskgraph_present_absent.c
index edc764a0fa2464..d06617b3f8aa7f 100644
--- a/offload/test/offloading/taskgraph_present_absent.c
+++ b/offload/test/offloading/taskgraph_present_absent.c
@@ -9,9 +9,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// `present` says the region is expected to be mapped already and this map must
// not bring it in. A graph cannot settle that at record time: whether a region
diff --git a/offload/test/offloading/taskgraph_target_data_body.c b/offload/test/offloading/taskgraph_target_data_body.c
index 21d743d7c891ce..25e967d7378812 100644
--- a/offload/test/offloading/taskgraph_target_data_body.c
+++ b/offload/test/offloading/taskgraph_target_data_body.c
@@ -7,9 +7,6 @@
// REQUIRES: gpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// OpenMP 6.1 [21.5]: the structured block of a 'target data' is a task of its
// own, ordered between the enter-data and exit-data tasks. A taskgraph
diff --git a/offload/test/offloading/taskgraph_target_data_body_visible.c b/offload/test/offloading/taskgraph_target_data_body_visible.c
index 523a850e7eb26a..bb18b5c5d78fcf 100644
--- a/offload/test/offloading/taskgraph_target_data_body_visible.c
+++ b/offload/test/offloading/taskgraph_target_data_body_visible.c
@@ -7,9 +7,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A graph holding the body of a 'target data' is replayed with the target
// constructs either side of the body on the device, as subgraphs, and the body
diff --git a/offload/test/offloading/taskgraph_target_encounter_order.c b/offload/test/offloading/taskgraph_target_encounter_order.c
new file mode 100644
index 00000000000000..8671fdce436b3c
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_encounter_order.c
@@ -0,0 +1,60 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: env LIBOMPTARGET_AMDGPU_TASKGRAPH=0 \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Encounter order of recorded target constructs is itself a dependence when the
+// construct carries no `nowait`: such a construct generates an undeferred
+// (included) task, which suspends the encountering task until it completes, so
+// on the recording execution the three constructs below necessarily ran one
+// after another. A replay has to preserve that.
+//
+// Nothing here carries a depend clause, so a replay that only honors depend
+// clauses is free to run the three nodes concurrently -- and the AMDGPU plugin
+// backend does exactly that, which used to read `a` back before the kernel had
+// written it and miscompile every replay after the first. The host replay path
+// happened to survive it by walking a parallel region's children one at a time,
+// so both replay paths are run here.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 1024;
+ const int Iters = 3;
+ int *a = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i)
+ a[i] = i;
+
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target enter data map(to : a[0 : N])
+
+#pragma omp target
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] + 5;
+ }
+
+#pragma omp target exit data map(from : a[0 : N])
+ }
+ }
+
+ int errors = 0;
+ for (int i = 0; i < N; ++i)
+ if (a[i] != i + 5 * Iters)
+ ++errors;
+
+ // CHECK: taskgraph target encounter order errors: 0
+ printf("taskgraph target encounter order errors: %d\n", errors);
+ free(a);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_target_mutex_irreducible.c b/offload/test/offloading/taskgraph_target_mutex_irreducible.c
index 61bb6ff8896e1a..53ba6c29b11508 100644
--- a/offload/test/offloading/taskgraph_target_mutex_irreducible.c
+++ b/offload/test/offloading/taskgraph_target_mutex_irreducible.c
@@ -9,9 +9,6 @@
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// The other half of taskgraph_target_mutex_exclusive.c: a `mutexinoutset` on a
// target that the region builder cannot express structurally, so the mutex sets
diff --git a/offload/test/offloading/taskgraph_target_plugin_backend.c b/offload/test/offloading/taskgraph_target_plugin_backend.c
new file mode 100644
index 00000000000000..b2a9f495956322
--- /dev/null
+++ b/offload/test/offloading/taskgraph_target_plugin_backend.c
@@ -0,0 +1,94 @@
+// RUN: %libomptarget-compile-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-version=60 && \
+// RUN: %libomptarget-run-generic | %fcheck-generic
+
+// REQUIRES: gpu
+// REQUIRES: omp_taskgraph_experimental
+
+// Replay of a graph of target regions with the AMDGPU plugin's graph backend
+// enabled, so the graph is claimed at finalize and every replay is issued by the
+// plugin from its own lowering instead of by libomp on the host. The result must
+// be identical either way; a plugin that declines this graph is also fine, since
+// libomp then replays it, so this covers both paths reaching the same answer
+// rather than asserting which one ran.
+//
+// The shape is a diamond: one kernel produces `a`, two independent kernels
+// consume it into `b` and `c`, and a fourth joins them into `d`. So a backend
+// that spreads the graph over concurrent lanes has to both run the middle pair
+// concurrently and synchronize across lanes at the fork and the join, and the
+// transfers of each buffer have to be ordered against the kernels that produce
+// and consume it.
+//
+// The shape comes from the clauses, and both halves are needed: the depend
+// clauses supply the three edges, and `nowait` is what leaves everything else
+// unordered. Dropping `nowait` makes each recorded target an undeferred task,
+// which takes an encounter-order edge as well and flattens the diamond into a
+// four-node chain -- still the right answer, but with none of the concurrency
+// this test exists to exercise. Dropping the depend clauses instead leaves
+// four unordered regions and a wrong one.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+int main() {
+ const int N = 1024;
+ const int Iters = 4;
+ int *a = (int *)malloc(N * sizeof(int));
+ int *b = (int *)malloc(N * sizeof(int));
+ int *c = (int *)malloc(N * sizeof(int));
+ int *d = (int *)malloc(N * sizeof(int));
+ for (int i = 0; i < N; ++i) {
+ a[i] = i;
+ b[i] = 0;
+ c[i] = 0;
+ d[i] = 0;
+ }
+
+ for (int it = 0; it < Iters; ++it) {
+#pragma omp taskgraph
+ {
+#pragma omp target nowait map(tofrom : a[0 : N]) depend(out : a[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ a[i] = a[i] + 1;
+ }
+#pragma omp target nowait map(to : a[0 : N]) map(tofrom : b[0 : N]) \
+ depend(in : a[0 : N]) depend(out : b[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ b[i] = b[i] + a[i];
+ }
+#pragma omp target nowait map(to : a[0 : N]) map(tofrom : c[0 : N]) \
+ depend(in : a[0 : N]) depend(out : c[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ c[i] = c[i] + 2 * a[i];
+ }
+#pragma omp target nowait map(to : b[0 : N], c[0 : N]) map(from : d[0 : N]) \
+ depend(in : b[0 : N], c[0 : N])
+ {
+ for (int i = 0; i < N; ++i)
+ d[i] = b[i] + c[i];
+ }
+ }
+ }
+
+ // Encounter k (1-based) leaves a[i] == i + k, so b[i] sums those, c[i] is
+ // twice b[i], and d[i] is three times b[i].
+ const int Sum = Iters * (Iters + 1) / 2;
+ int errors = 0;
+ for (int i = 0; i < N; ++i) {
+ const int B = Iters * i + Sum;
+ if (a[i] != i + Iters || b[i] != B || c[i] != 2 * B || d[i] != 3 * B)
+ ++errors;
+ }
+
+ // CHECK: taskgraph plugin backend errors: 0
+ printf("taskgraph plugin backend errors: %d\n", errors);
+ free(d);
+ free(c);
+ free(b);
+ free(a);
+ return errors != 0;
+}
diff --git a/offload/test/offloading/taskgraph_unordered_copy_decline.c b/offload/test/offloading/taskgraph_unordered_copy_decline.c
index 75a787a8cc62e2..9716e400a5b900 100644
--- a/offload/test/offloading/taskgraph_unordered_copy_decline.c
+++ b/offload/test/offloading/taskgraph_unordered_copy_decline.c
@@ -5,9 +5,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// A `target enter data` and a `target` region reading the same data, with
// nothing ordering the two. Outside a graph, whichever maps first copies the
diff --git a/offload/test/offloading/taskgraph_unordered_copy_replay.c b/offload/test/offloading/taskgraph_unordered_copy_replay.c
index 3213788957466c..749edd4353d3b6 100644
--- a/offload/test/offloading/taskgraph_unordered_copy_replay.c
+++ b/offload/test/offloading/taskgraph_unordered_copy_replay.c
@@ -5,9 +5,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// Two unordered `target` regions each mapping all of T `tofrom` and writing
// half of it. When the graph maps T itself, each copies T in and out, and
diff --git a/offload/test/offloading/taskgraph_use_device_ptr.c b/offload/test/offloading/taskgraph_use_device_ptr.c
index 8837233fab60f0..c4d5e1a1752690 100644
--- a/offload/test/offloading/taskgraph_use_device_ptr.c
+++ b/offload/test/offloading/taskgraph_use_device_ptr.c
@@ -10,9 +10,6 @@
// REQUIRES: gpu
// REQUIRES: amdgpu
// REQUIRES: omp_taskgraph_experimental
-// Fails until a plugin claims taskgraphs, which the AMDGPU backend is the
-// first to do.
-// XFAIL: *
// The body of a recorded 'target data' is a task, which converts the
// use_device_ptr list item itself whenever it starts. So each replay sees the
>From f2754feb5423a09462bf5203e2674db122c68236 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 29 Oct 2026 12:46:48 -0500
Subject: [PATCH 29/29] [skip ci] Overview snapshot of the taskgraph series
This empty commit only keeps CI from running on the overview PR, which
carries the whole series for reading and building. The series is reviewed
and tested through its individual PRs.
More information about the Openmp-commits
mailing list