[llvm] [openmp] [OFFLOAD][OPENMP] 6.0 compatible interop interface (PR #143491)

[Kevin Sala Penades via llvm-commits]

================
@@ -193,119 +194,276 @@ __OMP_GET_INTEROP_TY3(const char *, type_desc)
 __OMP_GET_INTEROP_TY3(const char *, rc_desc)
 #undef __OMP_GET_INTEROP_TY3
 
-static const char *copyErrorString(llvm::Error &&Err) {
-  // TODO: Use the error string while avoiding leaks.
-  std::string ErrMsg = llvm::toString(std::move(Err));
-  char *UsrMsg = reinterpret_cast<char *>(malloc(ErrMsg.size() + 1));
-  strcpy(UsrMsg, ErrMsg.c_str());
-  return UsrMsg;
-}
-
 extern "C" {
 
-void __tgt_interop_init(ident_t *LocRef, int32_t Gtid,
-                        omp_interop_val_t *&InteropPtr,
-                        kmp_interop_type_t InteropType, int32_t DeviceId,
-                        int32_t Ndeps, kmp_depend_info_t *DepList,
-                        int32_t HaveNowait) {
-  int32_t NdepsNoalias = 0;
-  kmp_depend_info_t *NoaliasDepList = NULL;
-  assert(InteropType != kmp_interop_type_unknown &&
-         "Cannot initialize with unknown interop_type!");
-  if (DeviceId == -1) {
-    DeviceId = omp_get_default_device();
+omp_interop_val_t *__tgt_interop_get(ident_t *LocRef, int32_t InteropType,
+                                     int64_t DeviceNum, int32_t NumPrefers,
+                                     interop_spec_t *Prefers,
+                                     interop_ctx_t *Ctx, dep_pack_t *Deps) {
+
+  DP("Call to %s with device_num %" PRId64 ", interop type %" PRId32
+     ", number of preferred specs %" PRId32 "%s%s\n",
+     __func__, DeviceNum, InteropType, NumPrefers,
+     Ctx->flags.implicit ? " (implicit)" : "",
+     Ctx->flags.nowait ? " (nowait)" : "");
+
+  if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED)
+    return omp_interop_none;
+
+  // Now, try to create an interop with device_num.
+  if (DeviceNum == OFFLOAD_DEVICE_DEFAULT)
+    DeviceNum = omp_get_default_device();
+
+  auto gtid = Ctx->gtid;
+
+  if (InteropType == kmp_interop_type_targetsync) {
+    if (Ctx->flags.nowait)
+      DP("Warning: nowait flag on interop creation not supported yet. "
+         "Ignored\n");
+    if (Deps)
+      __kmpc_omp_wait_deps(LocRef, gtid, Deps->ndeps, Deps->deplist,
+                           Deps->ndeps_noalias, Deps->noalias_deplist);
   }
 
-  if (InteropType == kmp_interop_type_tasksync) {
-    __kmpc_omp_wait_deps(LocRef, Gtid, Ndeps, DepList, NdepsNoalias,
-                         NoaliasDepList);
+  auto DeviceOrErr = PM->getDevice(DeviceNum);
+  if (!DeviceOrErr) {
+    [[maybe_unused]] std::string ErrStr = toString(DeviceOrErr.takeError());
+    DP("Couldn't find device %" PRId64
+       " while constructing interop object: %s\n",
+       DeviceNum, ErrStr.c_str());
+    return omp_interop_none;
+  }
+  auto &Device = *DeviceOrErr;
+  omp_interop_val_t *Interop = omp_interop_none;
+  auto InteropSpec = Device.RTL->select_interop_preference(
+      DeviceNum, InteropType, NumPrefers, Prefers);
+  if (InteropSpec.fr_id == omp_fr_none) {
+    DP("Interop request not supported by device %" PRId64 "\n", DeviceNum);
+    return omp_interop_none;
+  }
+  DP("Selected interop preference is fr_id=%s%s impl_attrs=%" PRId64 "\n",
+     getForeignRuntimeIdToStr((omp_foreign_runtime_id_t)InteropSpec.fr_id),
+     InteropSpec.attrs.inorder ? " inorder" : "", InteropSpec.impl_attrs);
+
+  if (Ctx->flags.implicit) {
+    // This is a request for an RTL managed interop object.
+    // Get it from the InteropTbl if possible
+    if (PM->InteropTbl.size() > 0) {
+      for (auto iop : PM->InteropTbl) {
+        if (iop->isCompatibleWith(InteropType, InteropSpec, DeviceNum, gtid)) {
+          Interop = iop;
+          Interop->markDirty();
+          DP("Reused interop " DPxMOD " from device number %" PRId64
+             " for gtid %" PRId32 "\n",
+             DPxPTR(Interop), DeviceNum, gtid);
+          return Interop;
+        }
+      }
+    }
   }
 
-  InteropPtr = new omp_interop_val_t(DeviceId, InteropType);
-
-  auto DeviceOrErr = PM->getDevice(DeviceId);
-  if (!DeviceOrErr) {
-    InteropPtr->err_str = copyErrorString(DeviceOrErr.takeError());
-    return;
+  Interop = Device.RTL->create_interop(DeviceNum, InteropType, &InteropSpec);
+  DP("Created an interop " DPxMOD " from device number %" PRId64 "\n",
+     DPxPTR(Interop), DeviceNum);
+
+  if (Ctx->flags.implicit) {
+    // register the new implicit interop in the RTL
+    Interop->setOwner(gtid);
+    Interop->markDirty();
+    PM->InteropTbl.add(Interop);
+  } else {
+    Interop->setOwner(-1);
   }
 
-  DeviceTy &Device = *DeviceOrErr;
-  if (!Device.RTL ||
-      Device.RTL->init_device_info(DeviceId, &(InteropPtr)->device_info,
-                                   &(InteropPtr)->err_str)) {
-    delete InteropPtr;
-    InteropPtr = omp_interop_none;
+  return Interop;
+}
+
+int __tgt_interop_use(ident_t *LocRef, omp_interop_val_t *Interop,
+                      interop_ctx_t *Ctx, dep_pack_t *Deps) {
+  bool nowait = Ctx->flags.nowait;
+  DP("Call to %s with interop " DPxMOD ", nowait %" PRId32 "\n", __func__,
+     DPxPTR(Interop), nowait);
+  if (OffloadPolicy::get(*PM).Kind == OffloadPolicy::DISABLED || !Interop)
+    return OFFLOAD_FAIL;
+
+  if (Interop->interop_type == kmp_interop_type_targetsync) {
+    if (Deps) {
+      if (nowait) {
+        DP("Warning: nowait flag on interop use with dependences not supported"
+           "yet. Ignored\n");
+        nowait = false;
+      }
+
+      __kmpc_omp_wait_deps(LocRef, Ctx->gtid, Deps->ndeps, Deps->deplist,
+                           Deps->ndeps_noalias, Deps->noalias_deplist);
+    }
   }
-  if (InteropType == kmp_interop_type_tasksync) {
-    if (!Device.RTL ||
-        Device.RTL->init_async_info(DeviceId, &(InteropPtr)->async_info)) {
-      delete InteropPtr;
-      InteropPtr = omp_interop_none;
+
+  if (Interop->async_info && Interop->async_info->Queue) {
+    if (nowait)
+      Interop->asyncBarrier();
+    else {
+      Interop->flush();
+      Interop->syncBarrier();
+      Interop->markClean();
----------------
kevinsala wrote:

If I understood correctly, each of these functions will try to get the device. Can we just get the device from the interop object and pass it to each of these devices?

https://github.com/llvm/llvm-project/pull/143491